From 918436fd5138988885f6b95535629beb2b32337f Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 13 Sep 2022 17:00:35 +0800 Subject: [PATCH 01/10] cmd, core, eth, les, light: track deleted nodes --- cmd/geth/dbcmd.go | 30 ++++--- cmd/geth/snapshot.go | 8 +- core/blockchain.go | 13 +-- core/state/database.go | 8 +- core/state/iterator.go | 2 +- core/state/metrics.go | 14 ++-- core/state/pruner/pruner.go | 6 +- core/state/snapshot/generate.go | 20 ++--- core/state/snapshot/generate_test.go | 4 +- core/state/state_object.go | 4 +- core/state/statedb.go | 49 +++++++---- core/state/sync_test.go | 6 +- core/state/trie_prefetcher.go | 8 +- eth/api.go | 4 +- eth/protocols/snap/handler.go | 10 +-- eth/protocols/snap/sync_test.go | 20 ++--- les/downloader/downloader_test.go | 2 +- les/handler_test.go | 8 +- les/server_handler.go | 4 +- les/server_requests.go | 2 +- light/odr.go | 12 ++- light/odr_test.go | 2 +- light/postprocess.go | 89 ++++++++++---------- light/trie.go | 9 +-- tests/fuzzers/trie/trie-fuzzer.go | 2 +- trie/committer.go | 32 +++++++- trie/database.go | 32 +++++++- trie/iterator.go | 14 +++- trie/iterator_test.go | 16 ++-- trie/nodeset.go | 117 +++++++++++++++++++++++---- trie/proof.go | 3 +- trie/secure_trie.go | 8 +- trie/secure_trie_test.go | 6 +- trie/sync_test.go | 10 +-- trie/trie.go | 49 +++++------ trie/trie_reader.go | 106 ++++++++++++++++++++++++ trie/trie_test.go | 56 +++++++++---- trie/util_test.go | 2 +- trie/utils.go | 38 ++++----- 39 files changed, 568 insertions(+), 257 deletions(-) create mode 100644 trie/trie_reader.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index bb53a632e86..8f24ef91ee5 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -150,7 +150,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Action: dbDumpTrie, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", - ArgsUsage: " ", + ArgsUsage: " ", Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -486,7 +486,7 @@ func dbPut(ctx *cli.Context) error { // dbDumpTrie shows the key-value slots of a given storage trie func dbDumpTrie(ctx *cli.Context) error { - if ctx.NArg() < 1 { + if ctx.NArg() < 2 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) @@ -495,29 +495,33 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() var ( - root []byte - start []byte - max = int64(-1) - err error + state []byte + storage []byte + start []byte + max = int64(-1) + err error ) - if root, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { + if state, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { log.Info("Could not decode the root", "error", err) return err } - stRoot := common.BytesToHash(root) - if ctx.NArg() >= 2 { - if start, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + if storage, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + log.Info("Could not decode the root", "error", err) + return err + } + if ctx.NArg() >= 3 { + if start, err = hexutil.Decode(ctx.Args().Get(2)); err != nil { log.Info("Could not decode the seek position", "error", err) return err } } - if ctx.NArg() >= 3 { - if max, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { + if ctx.NArg() >= 4 { + if max, err = strconv.ParseInt(ctx.Args().Get(3), 10, 64); err != nil { log.Info("Could not decode the max count", "error", err) return err } } - theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db)) + theTrie, err := trie.New(common.BytesToHash(state), common.Hash{}, common.BytesToHash(storage), trie.NewDatabase(db)) if err != nil { return err } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 39bef1f2d35..637c6fa6863 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -271,7 +271,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewStateTrie(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(root, common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -292,7 +292,7 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.Key), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIter.Key), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -360,7 +360,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewStateTrie(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(root, common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -406,7 +406,7 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") diff --git a/core/blockchain.go b/core/blockchain.go index fe127b5ea8a..9a1396c546b 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -70,6 +70,8 @@ var ( snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil) snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil) + triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil) + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) @@ -730,10 +732,10 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewStateTrie(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { - return err + root := block.Root() + if !bc.HasState(root) { + return fmt.Errorf("non existent state [%x..]", root[:4]) } - // If all checks out, manually set the head block. if !bc.chainmu.TryLock() { return errChainStopped @@ -745,7 +747,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { // Destroy any existing state snapshot and regenerate it in the background, // also resuming the normal maintenance of any previously paused snapshot. if bc.snaps != nil { - bc.snaps.Rebuild(block.Root()) + bc.snaps.Rebuild(root) } log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil @@ -1743,8 +1745,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them + triedbCommitTimer.Update(statedb.TrieDBCommits) // Triedb commits are complete, we can mark them - blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) + blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits - statedb.TrieDBCommits) blockInsertTimer.UpdateSince(start) // Report the import stats before returning the various results diff --git a/core/state/database.go b/core/state/database.go index 9b4fd8946e2..1aa901216b7 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -43,7 +43,7 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(addrHash, root common.Hash) (Trie, error) + OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie @@ -148,7 +148,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewStateTrie(common.Hash{}, root, db.db) + tr, err := trie.NewStateTrie(root, common.Hash{}, root, db.db) if err != nil { return nil, err } @@ -156,8 +156,8 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewStateTrie(addrHash, root, db.db) +func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) { + tr, err := trie.NewStateTrie(stateRoot, addrHash, root, db.db) if err != nil { return nil, err } diff --git a/core/state/iterator.go b/core/state/iterator.go index 611df52431e..ba7efd4653b 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -109,7 +109,7 @@ func (it *NodeIterator) step() error { if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } - dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root) + dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root) if err != nil { return err } diff --git a/core/state/metrics.go b/core/state/metrics.go index 35d2df92dda..e702ef3a81a 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -19,10 +19,12 @@ package state import "github.com/ethereum/go-ethereum/metrics" var ( - accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) - storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) - accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) - storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) - accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil) - storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil) + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil) + storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) + accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) + storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) ) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 87bc357a5c1..46b9481b55e 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -87,7 +87,7 @@ type Pruner struct { func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { headBlock := rawdb.ReadHeadBlock(db) if headBlock == nil { - return nil, errors.New("Failed to load head block") + return nil, errors.New("failed to load head block") } snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) if err != nil { @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(genesis.Root(), common.Hash{}, genesis.Root(), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index bf714db4c2d..310931e4b6e 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error) // // The proof result will be returned if the range proving is finished, otherwise // the error will be returned to abort the entire procedure. -func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { +func (dl *diskLayer) proveRange(ctx *generatorContext, stateRoot common.Hash, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { var ( keys [][]byte vals [][]byte @@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root c return &proofResult{keys: keys, vals: vals}, nil } // Snap state is chunked, generate edge proofs for verification. - tr, err := trie.New(owner, root, dl.triedb) + tr, err := trie.New(stateRoot, owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie @@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error // generateRange generates the state segment with particular prefix. Generation can // either verify the correctness of existing state through range-proof and skip // generation, or iterate trie to regenerate state on demand. -func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { +func (dl *diskLayer) generateRange(ctx *generatorContext, state common.Hash, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { // Use range prover to check the validity of the flat state in the range - result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn) + result, err := dl.proveRange(ctx, state, owner, root, prefix, kind, origin, max, valueConvertFn) if err != nil { return false, nil, err } @@ -363,7 +363,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb) + snapTrie, _ := trie.New(common.Hash{}, owner, common.Hash{}, snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } @@ -377,7 +377,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo // if it's already opened with some nodes resolved. tr := result.tr if tr == nil { - tr, err = trie.New(owner, root, dl.triedb) + tr, err = trie.New(state, owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return false, nil, errMissingTrie @@ -511,7 +511,7 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error // generateStorages generates the missing storage slots of the specific contract. // It's supposed to restart the generation from the given origin position. -func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, storageRoot common.Hash, storeMarker []byte) error { +func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Hash, account common.Hash, storageRoot common.Hash, storeMarker []byte) error { onStorage := func(key []byte, val []byte, write bool, delete bool) error { defer func(start time.Time) { snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds()) @@ -540,7 +540,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, // Loop for re-generating the missing storage slots. var origin = common.CopyBytes(storeMarker) for { - exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + exhausted, last, err := dl.generateRange(ctx, stateRoot, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } @@ -624,7 +624,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er if accMarker != nil && bytes.Equal(account[:], accMarker) && len(dl.genMarker) > common.HashLength { storeMarker = dl.genMarker[common.HashLength:] } - if err := generateStorages(ctx, dl, account, acc.Root, storeMarker); err != nil { + if err := generateStorages(ctx, dl, dl.root, account, acc.Root, storeMarker); err != nil { return err } } @@ -640,7 +640,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er } origin := common.CopyBytes(accMarker) for { - exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + exhausted, last, err := dl.generateRange(ctx, dl.root, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 58cfb464ff7..8c6ac4d0ca2 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -149,7 +149,7 @@ type testHelper struct { func newHelper() *testHelper { diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, @@ -182,7 +182,7 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { - stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb) + stTrie, _ := trie.NewStateTrie(stateRoot, owner, common.Hash{}, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } diff --git a/core/state/state_object.go b/core/state/state_object.go index a23df895458..178b9305931 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -159,9 +159,9 @@ func (s *stateObject) getTrie(db Database) Trie { } if s.trie == nil { var err error - s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) + s.trie, err = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root) if err != nil { - s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) + s.trie, _ = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, common.Hash{}) s.setError(fmt.Errorf("can't create storage trie: %v", err)) } } diff --git a/core/state/statedb.go b/core/state/statedb.go index b05f1742f57..29a1ccf2d73 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -120,6 +120,7 @@ type StateDB struct { SnapshotAccountReads time.Duration SnapshotStorageReads time.Duration SnapshotCommits time.Duration + TrieDBCommits time.Duration AccountUpdated int StorageUpdated int @@ -904,9 +905,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // Commit objects to the trie, measuring the elapsed time var ( - accountTrieNodes int - storageTrieNodes int - nodes = trie.NewMergedNodeSet() + accountTrieNodesUpdated int + accountTrieNodesDeleted int + storageTrieNodesUpdated int + storageTrieNodesDeleted int + nodes = trie.NewMergedNodeSet() ) codeWriter := s.db.DiskDB().NewBatch() for addr := range s.stateObjectsDirty { @@ -926,7 +929,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if err := nodes.Merge(set); err != nil { return common.Hash{}, err } - storageTrieNodes += set.Len() + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted } } } @@ -952,7 +957,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if err := nodes.Merge(set); err != nil { return common.Hash{}, err } - accountTrieNodes = set.Len() + accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -961,16 +966,16 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { storageUpdatedMeter.Mark(int64(s.StorageUpdated)) accountDeletedMeter.Mark(int64(s.AccountDeleted)) storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountTrieCommittedMeter.Mark(int64(accountTrieNodes)) - storageTriesCommittedMeter.Mark(int64(storageTrieNodes)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) s.AccountUpdated, s.AccountDeleted = 0, 0 s.StorageUpdated, s.StorageDeleted = 0, 0 } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { - if metrics.EnabledExpensive { - defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now()) - } + start := time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil { @@ -984,13 +989,29 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) } } + if metrics.EnabledExpensive { + s.SnapshotCommits += time.Since(start) + } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } - if err := s.db.TrieDB().Update(nodes); err != nil { - return common.Hash{}, err + if root == (common.Hash{}) { + root = emptyRoot + } + origin := s.originalRoot + if origin == (common.Hash{}) { + origin = emptyRoot + } + if root != origin { + start := time.Now() + if err := s.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } + s.originalRoot = root + if metrics.EnabledExpensive { + s.TrieDBCommits += time.Since(start) + } } - s.originalRoot = root - return root, err + return root, nil } // PrepareAccessList handles the preparatory steps for executing a state transition with diff --git a/core/state/sync_test.go b/core/state/sync_test.go index d16c7ce7322..1e73c54ab8a 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.KeyValueStore, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db)) + trie, err := trie.New(root, common.Hash{}, root, trie.NewDatabase(db)) if err != nil { return err } @@ -174,7 +174,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if commit { srcDb.TrieDB().Commit(srcRoot, false, nil) } - srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB()) + srcTrie, _ := trie.New(srcRoot, common.Hash{}, srcRoot, srcDb.TrieDB()) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -222,7 +222,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) } - stTrie, err := trie.New(common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB()) + stTrie, err := trie.New(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB()) if err != nil { t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 678774a62b7..2e16f587ce5 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -150,7 +150,7 @@ func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][] id := p.trieID(owner, root) fetcher := p.fetchers[id] if fetcher == nil { - fetcher = newSubfetcher(p.db, owner, root) + fetcher = newSubfetcher(p.db, p.root, owner, root) p.fetchers[id] = fetcher } fetcher.schedule(keys) @@ -206,6 +206,7 @@ func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { // the trie being worked on is retrieved from the prefetcher. type subfetcher struct { db Database // Database to load trie nodes through + state common.Hash // Root hash of the state to prefetch owner common.Hash // Owner of the trie, usually account hash root common.Hash // Root hash of the trie to prefetch trie Trie // Trie being populated with nodes @@ -225,9 +226,10 @@ type subfetcher struct { // newSubfetcher creates a goroutine to prefetch state items belonging to a // particular root hash. -func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher { +func newSubfetcher(db Database, state common.Hash, owner common.Hash, root common.Hash) *subfetcher { sf := &subfetcher{ db: db, + state: state, owner: owner, root: root, wake: make(chan struct{}, 1), @@ -298,7 +300,7 @@ func (sf *subfetcher) loop() { } sf.trie = trie } else { - trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root) + trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root) if err != nil { log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) return diff --git a/eth/api.go b/eth/api.go index 3b5bb5f0aa9..556ee10c363 100644 --- a/eth/api.go +++ b/eth/api.go @@ -508,11 +508,11 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewStateTrie(common.Hash{}, startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(startBlock.Root(), common.Hash{}, startBlock.Root(), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewStateTrie(common.Hash{}, endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(endBlock.Root(), common.Hash{}, endBlock.Root(), triedb) if err != nil { return nil, err } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 41380d96f57..c35cd68d358 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -283,7 +283,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) + tr, err := trie.New(req.Root, common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -413,7 +413,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(req.Root, common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -421,7 +421,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.NewStateTrie(account, acc.Root, chain.StateCache().TrieDB()) + stTrie, err := trie.NewStateTrie(req.Root, account, acc.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -487,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, triedb) + accTrie, err := trie.NewStateTrie(req.Root, common.Hash{}, req.Root, triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -529,7 +529,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewStateTrie(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) + stTrie, err := trie.NewStateTrie(req.Root, common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 45124570656..ee6288e4b53 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1372,7 +1372,7 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { root, nodes, _ := accTrie.Commit(false) db.Update(trie.NewWithNodeSet(nodes)) - accTrie, _ = trie.New(common.Hash{}, root, db) + accTrie, _ = trie.New(root, common.Hash{}, root, db) return accTrie, entries } @@ -1434,7 +1434,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { root, nodes, _ := accTrie.Commit(false) db.Update(trie.NewWithNodeSet(nodes)) - accTrie, _ = trie.New(common.Hash{}, root, db) + accTrie, _ = trie.New(root, common.Hash{}, root, db) return accTrie, entries } @@ -1484,10 +1484,10 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) db.Update(nodes) // Re-create tries with new root - accTrie, _ = trie.New(common.Hash{}, root, db) + accTrie, _ = trie.New(root, common.Hash{}, root, db) for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) - trie, _ := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + trie, _ := trie.New(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) storageTries[common.BytesToHash(key)] = trie } return accTrie, entries, storageTries, storageEntries @@ -1548,13 +1548,13 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie db.Update(nodes) // Re-create tries with new root - accTrie, err := trie.New(common.Hash{}, root, db) + accTrie, err := trie.New(root, common.Hash{}, root, db) if err != nil { panic(err) } for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) - trie, err := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + trie, err := trie.New(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) if err != nil { panic(err) } @@ -1567,7 +1567,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { - trie, _ := trie.New(owner, common.Hash{}, db) + trie, _ := trie.New(common.Hash{}, owner, common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1593,7 +1593,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(owner, common.Hash{}, db) + trie, _ = trie.New(common.Hash{}, owner, common.Hash{}, db) ) // Initialize boundaries var next common.Hash @@ -1640,7 +1640,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(db) - accTrie, err := trie.New(common.Hash{}, root, triedb) + accTrie, err := trie.New(root, common.Hash{}, root, triedb) if err != nil { t.Fatal(err) } @@ -1658,7 +1658,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewStateTrie(common.BytesToHash(accIt.Key), acc.Root, triedb) + storeTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIt.Key), acc.Root, triedb) if err != nil { t.Fatal(err) } diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index c56870ff178..9a9aa0a8131 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewStateTrie(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(block.Root(), common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/les/handler_test.go b/les/handler_test.go index 56d7d55b5a5..cf66184247e 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) + trie, _ := trie.New(header.Root, common.Hash{}, header.Root, trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) + t, _ := trie.New(header.Root, common.Hash{}, header.Root, trie.NewDatabase(server.db)) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/server_handler.go b/les/server_handler.go index a199a34a725..b58926009c4 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool { // getAccount retrieves an account from the state based on root. func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(common.Hash{}, root, triedb) + trie, err := trie.New(root, common.Hash{}, root, triedb) if err != nil { return types.StateAccount{}, err } @@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) return trie } diff --git a/les/server_requests.go b/les/server_requests.go index bab5f733d54..b0eb2371e02 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -428,7 +428,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { p.bumpInvalid() continue } - trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root) + trie, err = statedb.OpenStorageTrie(root, common.BytesToHash(request.AccKey), account.Root) if trie == nil || err != nil { p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) continue diff --git a/light/odr.go b/light/odr.go index 9521dd53e85..7cebe010d41 100644 --- a/light/odr.go +++ b/light/odr.go @@ -53,9 +53,11 @@ type OdrRequest interface { // TrieID identifies a state or account storage trie type TrieID struct { - BlockHash, Root common.Hash - BlockNumber uint64 - AccKey []byte + BlockHash common.Hash + BlockNumber uint64 + StateRoot common.Hash + Root common.Hash + AccKey []byte } // StateTrieID returns a TrieID for a state trie belonging to a certain block @@ -64,8 +66,9 @@ func StateTrieID(header *types.Header) *TrieID { return &TrieID{ BlockHash: header.Hash(), BlockNumber: header.Number.Uint64(), - AccKey: nil, + StateRoot: header.Root, Root: header.Root, + AccKey: nil, } } @@ -76,6 +79,7 @@ func StorageTrieID(state *TrieID, addrHash, root common.Hash) *TrieID { return &TrieID{ BlockHash: state.BlockHash, BlockNumber: state.BlockNumber, + StateRoot: state.StateRoot, AccKey: addrHash[:], Root: root, } diff --git a/light/odr_test.go b/light/odr_test.go index 7f567058754..903c7f6f90a 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -87,7 +87,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { t state.Trie ) if len(req.Id.AccKey) > 0 { - t, err = odr.serverState.OpenStorageTrie(common.BytesToHash(req.Id.AccKey), req.Id.Root) + t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToHash(req.Id.AccKey), req.Id.Root) } else { t, err = odr.serverState.OpenTrie(req.Id.Root) } diff --git a/light/postprocess.go b/light/postprocess.go index 3f9da659333..f19c1e8861d 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -25,7 +25,6 @@ import ( "math/big" "time" - mapset "github.com/deckarep/golang-set" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" @@ -134,7 +133,6 @@ type ChtIndexerBackend struct { diskdb, trieTable ethdb.Database odr OdrBackend triedb *trie.Database - trieset mapset.Set section, sectionSize uint64 lastHash common.Hash trie *trie.Trie @@ -148,7 +146,6 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis odr: odr, trieTable: trieTable, triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down - trieset: mapset.NewSet(), sectionSize: size, disablePruning: disablePruning, } @@ -187,12 +184,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error - c.trie, err = trie.New(common.Hash{}, root, c.triedb) + c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) if err != nil && c.odr != nil { err = c.fetchMissingNodes(ctx, section, root) if err == nil { - c.trie, err = trie.New(common.Hash{}, root, c.triedb) + c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) } } c.section = section @@ -226,38 +223,44 @@ func (c *ChtIndexerBackend) Commit() error { if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { return err } + if err := c.triedb.Commit(root, false, nil); err != nil { + return err + } } // Re-create trie with newly generated root and updated database. - c.trie, err = trie.New(common.Hash{}, root, c.triedb) + c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) if err != nil { return err } // Pruning historical trie nodes if necessary. if !c.disablePruning { - // Flush the triedb and track the latest trie nodes. - c.trieset.Clear() - c.triedb.Commit(root, false, func(hash common.Hash) { c.trieset.Add(hash) }) - it := c.trieTable.NewIterator(nil, nil) defer it.Release() var ( - deleted int - remaining int - t = time.Now() + deleted int + batch = c.trieTable.NewBatch() + t = time.Now() ) + hashes := make(map[common.Hash]struct{}) + if nodes != nil { + for _, hash := range nodes.Hashes() { + hashes[hash] = struct{}{} + } + } for it.Next() { trimmed := bytes.TrimPrefix(it.Key(), []byte(ChtTablePrefix)) - if !c.trieset.Contains(common.BytesToHash(trimmed)) { - c.trieTable.Delete(trimmed) - deleted += 1 - } else { - remaining += 1 + if len(trimmed) == common.HashLength { + if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { + batch.Delete(trimmed) + deleted += 1 + } } } - log.Debug("Prune historical CHT trie nodes", "deleted", deleted, "remaining", remaining, "elapsed", common.PrettyDuration(time.Since(t))) - } else { - c.triedb.Commit(root, false, nil) + if err := batch.Write(); err != nil { + return err + } + log.Debug("Prune historical CHT trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) } log.Info("Storing CHT", "section", c.section, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) StoreChtRoot(c.diskdb, c.section, c.lastHash, root) @@ -333,7 +336,6 @@ type BloomTrieIndexerBackend struct { disablePruning bool diskdb, trieTable ethdb.Database triedb *trie.Database - trieset mapset.Set odr OdrBackend section uint64 parentSize uint64 @@ -351,7 +353,6 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin odr: odr, trieTable: trieTable, triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down - trieset: mapset.NewSet(), parentSize: parentSize, size: size, disablePruning: disablePruning, @@ -414,11 +415,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error - b.trie, err = trie.New(common.Hash{}, root, b.triedb) + b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) if err != nil && b.odr != nil { err = b.fetchMissingNodes(ctx, section, root) if err == nil { - b.trie, err = trie.New(common.Hash{}, root, b.triedb) + b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) } } b.section = section @@ -473,38 +474,44 @@ func (b *BloomTrieIndexerBackend) Commit() error { if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { return err } + if err := b.triedb.Commit(root, false, nil); err != nil { + return err + } } // Re-create trie with newly generated root and updated database. - b.trie, err = trie.New(common.Hash{}, root, b.triedb) + b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) if err != nil { return err } // Pruning historical trie nodes if necessary. if !b.disablePruning { - // Flush the triedb and track the latest trie nodes. - b.trieset.Clear() - b.triedb.Commit(root, false, func(hash common.Hash) { b.trieset.Add(hash) }) - it := b.trieTable.NewIterator(nil, nil) defer it.Release() var ( - deleted int - remaining int - t = time.Now() + deleted int + batch = b.trieTable.NewBatch() + t = time.Now() ) + hashes := make(map[common.Hash]struct{}) + if nodes != nil { + for _, hash := range nodes.Hashes() { + hashes[hash] = struct{}{} + } + } for it.Next() { trimmed := bytes.TrimPrefix(it.Key(), []byte(BloomTrieTablePrefix)) - if !b.trieset.Contains(common.BytesToHash(trimmed)) { - b.trieTable.Delete(trimmed) - deleted += 1 - } else { - remaining += 1 + if len(trimmed) == common.HashLength { + if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { + batch.Delete(trimmed) + deleted += 1 + } } } - log.Debug("Prune historical bloom trie nodes", "deleted", deleted, "remaining", remaining, "elapsed", common.PrettyDuration(time.Since(t))) - } else { - b.triedb.Commit(root, false, nil) + if err := batch.Write(); err != nil { + return err + } + log.Debug("Prune historical bloom trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) } sectionHead := b.sectionHeads[b.bloomTrieRatio-1] StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) diff --git a/light/trie.go b/light/trie.go index 0f2e3862553..12beed97cff 100644 --- a/light/trie.go +++ b/light/trie.go @@ -54,7 +54,7 @@ func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } -func (db *odrDatabase) OpenStorageTrie(addrHash, root common.Hash) (state.Trie, error) { +func (db *odrDatabase) OpenStorageTrie(state, addrHash, root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: StorageTrieID(db.id, addrHash, root)}, nil } @@ -63,8 +63,7 @@ func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { case *odrTrie: cpy := &odrTrie{db: t.db, id: t.id} if t.trie != nil { - cpytrie := *t.trie - cpy.trie = &cpytrie + cpy.trie = t.trie.Copy() } return cpy default: @@ -201,7 +200,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error { if len(t.id.AccKey) > 0 { owner = common.BytesToHash(t.id.AccKey) } - t.trie, err = trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) + t.trie, err = trie.New(t.id.StateRoot, owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) } if err == nil { err = fn() @@ -231,7 +230,7 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { if len(t.id.AccKey) > 0 { owner = common.BytesToHash(t.id.AccKey) } - t, err := trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) + t, err := trie.New(t.id.StateRoot, owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) if err == nil { it.t.trie = t } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 25e137602ca..52b8d9f1dc6 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -170,7 +170,7 @@ func runRandTest(rt randTest) error { return err } } - newtr, err := trie.New(common.Hash{}, hash, triedb) + newtr, err := trie.New(hash, common.Hash{}, hash, triedb) if err != nil { return err } diff --git a/trie/committer.go b/trie/committer.go index 28fc5a63f94..b241b34eecf 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -33,13 +33,15 @@ type leaf struct { // insertion order. type committer struct { nodes *NodeSet + tracer *tracer collectLeaf bool } // newCommitter creates a new committer or picks one from the pool. -func newCommitter(owner common.Hash, collectLeaf bool) *committer { +func newCommitter(owner common.Hash, tracer *tracer, collectLeaf bool) *committer { return &committer{ nodes: NewNodeSet(owner), + tracer: tracer, collectLeaf: collectLeaf, } } @@ -51,6 +53,20 @@ func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { if err != nil { return nil, nil, err } + // Some nodes can be deleted from trie which can't be captured by committer + // itself. Iterate all deleted nodes tracked by tracer and marked them as + // deleted only if they are present in database previously. + for _, path := range c.tracer.deleteList() { + // There are a few possibilities for this scenario(the node is deleted + // but not present in database previously), for example the node was + // embedded in the parent and now deleted from the trie. In this case + // it's noop from database's perspective. + val := c.tracer.getPrev(path) + if val == nil { + continue + } + c.nodes.markDeleted(path, val) + } return h.(hashNode), c.nodes, nil } @@ -83,6 +99,12 @@ func (c *committer) commit(path []byte, n node) (node, error) { if hn, ok := hashedNode.(hashNode); ok { return hn, nil } + // The short node now is embedded in its parent. Mark the node as + // deleted if it's present in database previously. It's equivalent + // as deletion from database's perspective. + if prev := c.tracer.getPrev(path); len(prev) != 0 { + c.nodes.markDeleted(path, prev) + } return collapsed, nil case *fullNode: hashedKids, err := c.commitChildren(path, cn) @@ -96,6 +118,12 @@ func (c *committer) commit(path []byte, n node) (node, error) { if hn, ok := hashedNode.(hashNode); ok { return hn, nil } + // The full node now is embedded in its parent. Mark the node as + // deleted if it's present in database previously. It's equivalent + // as deletion from database's perspective. + if prev := c.tracer.getPrev(path); len(prev) != 0 { + c.nodes.markDeleted(path, prev) + } return collapsed, nil case hashNode: return cn, nil @@ -161,7 +189,7 @@ func (c *committer) store(path []byte, n node) node { } ) // Collect the dirty node to nodeset for return. - c.nodes.add(string(path), mnode) + c.nodes.markUpdated(path, mnode, c.tracer.getPrev(path)) // Collect the corresponding leaf node if it's required. We don't check // full node since it's impossible to store value in fullNode. The key diff --git a/trie/database.go b/trie/database.go index 30120570ab5..76ca188add9 100644 --- a/trie/database.go +++ b/trie/database.go @@ -795,8 +795,8 @@ func (db *Database) Update(nodes *MergedNodeSet) error { } for _, owner := range order { subset := nodes.sets[owner] - for _, path := range subset.paths { - n, ok := subset.nodes[path] + for _, path := range subset.updates.order { + n, ok := subset.updates.nodes[path] if !ok { return fmt.Errorf("missing node %x %v", owner, path) } @@ -837,6 +837,34 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize } +// GetReader retrieves a node reader belonging to the given state root. +func (db *Database) GetReader(root common.Hash) Reader { + return newHashReader(db) +} + +// hashReader is reader of hashDatabase which implements the Reader interface. +type hashReader struct { + db *Database +} + +// newHashReader initializes the hash reader. +func newHashReader(db *Database) *hashReader { + return &hashReader{db: db} +} + +// Node retrieves the trie node with the given node hash. +// No error will be returned if the node is not found. +func (reader *hashReader) Node(_ common.Hash, _ []byte, hash common.Hash) (node, error) { + return reader.db.node(hash), nil +} + +// NodeBlob retrieves the RLP-encoded trie node blob with the given node hash. +// No error will be returned if the node is not found. +func (reader *hashReader) NodeBlob(_ common.Hash, _ []byte, hash common.Hash) ([]byte, error) { + blob, _ := reader.db.Node(hash) + return blob, nil +} + // saveCache saves clean state cache to given directory path // using specified CPU cores. func (db *Database) saveCache(dir string, threads int) error { diff --git a/trie/iterator.go b/trie/iterator.go index 1e76625c621..547a0d3a829 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -375,7 +375,12 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } } - return it.trie.resolveHash(hash, path) + // Retrieve the specified node from the underlying node reader. + // it.trie.resolveHash is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. + return it.trie.reader.node(it.trie.owner, path, common.BytesToHash(hash)) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { @@ -384,7 +389,12 @@ func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) return blob, nil } } - return it.trie.resolveBlob(hash, path) + // Retrieve the specified node from the underlying node reader. + // it.trie.resolveHash is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. + return it.trie.reader.nodeBlob(it.trie.owner, path, common.BytesToHash(hash)) } func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index e9d822a9a4f..46edbe0104a 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -66,7 +66,7 @@ func TestIterator(t *testing.T) { } db.Update(NewWithNodeSet(nodes)) - trie, _ = New(common.Hash{}, root, db) + trie, _ = New(root, common.Hash{}, root, db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -227,7 +227,7 @@ func TestDifferenceIterator(t *testing.T) { } rootA, nodesA, _ := triea.Commit(false) dba.Update(NewWithNodeSet(nodesA)) - triea, _ = New(common.Hash{}, rootA, dba) + triea, _ = New(rootA, common.Hash{}, rootA, dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) trieb := NewEmpty(dbb) @@ -236,7 +236,7 @@ func TestDifferenceIterator(t *testing.T) { } rootB, nodesB, _ := trieb.Commit(false) dbb.Update(NewWithNodeSet(nodesB)) - trieb, _ = New(common.Hash{}, rootB, dbb) + trieb, _ = New(rootB, common.Hash{}, rootB, dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -269,7 +269,7 @@ func TestUnionIterator(t *testing.T) { } rootA, nodesA, _ := triea.Commit(false) dba.Update(NewWithNodeSet(nodesA)) - triea, _ = New(common.Hash{}, rootA, dba) + triea, _ = New(rootA, common.Hash{}, rootA, dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) trieb := NewEmpty(dbb) @@ -278,7 +278,7 @@ func TestUnionIterator(t *testing.T) { } rootB, nodesB, _ := trieb.Commit(false) dbb.Update(NewWithNodeSet(nodesB)) - trieb, _ = New(common.Hash{}, rootB, dbb) + trieb, _ = New(rootB, common.Hash{}, rootB, dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -356,7 +356,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. - tr, _ := New(common.Hash{}, tr.Hash(), triedb) + tr, _ := New(tr.Hash(), common.Hash{}, tr.Hash(), triedb) // Remove a random node from the database. It can't be the root node // because that one is already loaded. @@ -445,7 +445,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - tr, _ := New(common.Hash{}, root, triedb) + tr, _ := New(root, common.Hash{}, root, triedb) it := tr.NodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { @@ -533,7 +533,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { diff --git a/trie/nodeset.go b/trie/nodeset.go index 08b9b35ebc8..8e0a1fe5150 100644 --- a/trie/nodeset.go +++ b/trie/nodeset.go @@ -18,6 +18,7 @@ package trie import ( "fmt" + "reflect" "github.com/ethereum/go-ethereum/common" ) @@ -25,18 +26,71 @@ import ( // memoryNode is all the information we know about a single cached trie node // in the memory. type memoryNode struct { - hash common.Hash // Node hash, computed by hashing rlp value - size uint16 // Byte size of the useful cached data - node node // Cached collapsed trie node, or raw rlp data + hash common.Hash // Node hash, computed by hashing rlp value, empty for deleted nodes + size uint16 // Byte size of the useful cached data, 0 for deleted nodes + node node // Cached collapsed trie node, or raw rlp data, nil for deleted nodes +} + +// memoryNodeSize is the raw size of a memoryNode data structure without any +// node data included. It's an approximate size, but should be a lot better +// than not counting them. +var memoryNodeSize = int(reflect.TypeOf(memoryNode{}).Size()) + +// memorySize returns the total memory size used by this node. +func (n *memoryNode) memorySize(key int) int { + return int(n.size) + memoryNodeSize + key +} + +// rlp returns the raw rlp encoded blob of the cached trie node, either directly +// from the cache, or by regenerating it from the collapsed node. +func (n *memoryNode) rlp() []byte { + if node, ok := n.node.(rawNode); ok { + return node + } + return nodeToBytes(n.node) +} + +// obj returns the decoded and expanded trie node, either directly from the cache, +// or by regenerating it from the rlp encoded blob. +func (n *memoryNode) obj() node { + if node, ok := n.node.(rawNode); ok { + return mustDecodeNode(n.hash[:], node) + } + return expandNode(n.hash[:], n.node) +} + +// nodeWithPrev wraps the memoryNode with the previous node value. +type nodeWithPrev struct { + *memoryNode + prev []byte // RLP-encoded previous value, nil means it's non-existent +} + +// unwrap returns the internal memoryNode object. +func (n *nodeWithPrev) unwrap() *memoryNode { + return n.memoryNode +} + +// memorySize returns the total memory size used by this node. It overloads +// the function in memoryNode by counting the size of previous value as well. +func (n *nodeWithPrev) memorySize(key int) int { + return n.memoryNode.memorySize(key) + len(n.prev) +} + +// nodesWithOrder represents a collection of dirty nodes which includes +// newly-inserted and updated nodes. The modification order of all nodes +// is represented by order list. +type nodesWithOrder struct { + order []string // the path list of dirty nodes, sort by insertion order + nodes map[string]*nodeWithPrev // the map of dirty nodes, keyed by node path } // NodeSet contains all dirty nodes collected during the commit operation. // Each node is keyed by path. It's not thread-safe to use. type NodeSet struct { - owner common.Hash // the identifier of the trie - paths []string // the path of dirty nodes, sort by insertion order - nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path - leaves []*leaf // the list of dirty leaves + owner common.Hash // the identifier of the trie + updates *nodesWithOrder // the set of updated nodes(newly inserted, updated) + deletes map[string][]byte // the map of deleted nodes, keyed by node + leaves []*leaf // the list of dirty leaves } // NewNodeSet initializes an empty node set to be used for tracking dirty nodes @@ -45,24 +99,55 @@ type NodeSet struct { func NewNodeSet(owner common.Hash) *NodeSet { return &NodeSet{ owner: owner, - nodes: make(map[string]*memoryNode), + updates: &nodesWithOrder{ + nodes: make(map[string]*nodeWithPrev), + }, + deletes: make(map[string][]byte), } } -// add caches node with provided path and node object. -func (set *NodeSet) add(path string, node *memoryNode) { - set.paths = append(set.paths, path) - set.nodes[path] = node +// NewNodeSetWithDeletion initializes the nodeset with provided deletion set. +func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *NodeSet { + set := NewNodeSet(owner) + for i, path := range paths { + set.markDeleted(path, prev[i]) + } + return set } -// addLeaf caches the provided leaf node. +// markUpdated marks the node as dirty(newly-inserted or updated) with provided +// node path, node object along with its previous value. +func (set *NodeSet) markUpdated(path []byte, node *memoryNode, prev []byte) { + set.updates.order = append(set.updates.order, string(path)) + set.updates.nodes[string(path)] = &nodeWithPrev{ + memoryNode: node, + prev: prev, + } +} + +// markDeleted marks the node as deleted with provided path and previous value. +func (set *NodeSet) markDeleted(path []byte, prev []byte) { + set.deletes[string(path)] = prev +} + +// addLeaf collects the provided leaf node into set. func (set *NodeSet) addLeaf(node *leaf) { set.leaves = append(set.leaves, node) } -// Len returns the number of dirty nodes contained in the set. -func (set *NodeSet) Len() int { - return len(set.nodes) +// Size returns the number of updated and deleted nodes contained in the set. +func (set *NodeSet) Size() (int, int) { + return len(set.updates.order), len(set.deletes) +} + +// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can +// we get rid of it? +func (set *NodeSet) Hashes() []common.Hash { + var ret []common.Hash + for _, node := range set.updates.nodes { + ret = append(ret, node.hash) + } + return ret } // MergedNodeSet represents a merged dirty node set for a group of tries. diff --git a/trie/proof.go b/trie/proof.go index 8c00bcf5329..11ad44fc8d5 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -22,7 +22,6 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -559,7 +558,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())} + tr := &Trie{root: root, reader: newEmptyReader()} if empty { tr.root = nil } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 0b7d33b2199..3f49480722d 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -29,8 +29,8 @@ type SecureTrie = StateTrie // NewSecure creates a new StateTrie. // Deprecated: use NewStateTrie. -func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { - return NewStateTrie(owner, root, db) +func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { + return NewStateTrie(stateRoot, owner, root, db) } // StateTrie wraps a trie with key hashing. In a stateTrie trie, all @@ -56,11 +56,11 @@ type StateTrie struct { // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil // and returns MissingNodeError if the root node cannot be found. -func NewStateTrie(owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { +func NewStateTrie(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { if db == nil { panic("trie.NewStateTrie called without a database") } - trie, err := New(owner, root, db) + trie, err := New(stateRoot, owner, root, db) if err != nil { return nil, err } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 862c3a3ec43..6b4839a5d7c 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -29,7 +29,7 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) return trie } @@ -37,7 +37,7 @@ func newEmptySecure() *StateTrie { func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -66,7 +66,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state - trie, _ = NewSecure(common.Hash{}, root, triedb) + trie, _ = NewStateTrie(root, common.Hash{}, root, triedb) return triedb, trie, content } diff --git a/trie/sync_test.go b/trie/sync_test.go index 9fd1d636c03..8b52bd352dc 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -30,7 +30,7 @@ import ( func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -59,7 +59,7 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state - trie, _ = NewSecure(common.Hash{}, root, triedb) + trie, _ = NewStateTrie(root, common.Hash{}, root, triedb) return triedb, trie, content } @@ -67,7 +67,7 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewStateTrie(common.Hash{}, common.BytesToHash(root), db) + trie, err := NewStateTrie(common.BytesToHash(root), common.Hash{}, common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -84,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewStateTrie(common.Hash{}, root, db) + trie, err := NewStateTrie(root, common.Hash{}, root, db) if err != nil { return nil // Consider a non existent state consistent } @@ -106,7 +106,7 @@ func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New()) emptyA := NewEmpty(dbA) - emptyB, _ := New(common.Hash{}, emptyRoot, dbB) + emptyB, _ := New(emptyRoot, common.Hash{}, emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) diff --git a/trie/trie.go b/trie/trie.go index 1ef1469c8dd..c29b4f40e1b 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -67,9 +67,8 @@ type Trie struct { // actually unhashed nodes. unhashed int - // db is the handler trie can retrieve nodes from. It's - // only for reading purpose and not available for writing. - db *Database + // reader is the handler trie can retrieve nodes from. + reader *trieReader // tracer is the tool to track the trie changes. // It will be reset after each commit operation. @@ -87,7 +86,7 @@ func (t *Trie) Copy() *Trie { root: t.root, owner: t.owner, unhashed: t.unhashed, - db: t.db, + reader: t.reader, tracer: t.tracer.copy(), } } @@ -99,10 +98,14 @@ func (t *Trie) Copy() *Trie { // trie is initially empty and does not require a database. Otherwise, // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. -func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { +func New(stateRoot common.Hash, owner common.Hash, root common.Hash, db NodeReader) (*Trie, error) { + reader, err := newTrieReader(owner, stateRoot, db) + if err != nil { + return nil, err + } trie := &Trie{ - owner: owner, - db: db, + owner: owner, + reader: reader, //tracer: newTracer(), } if root != (common.Hash{}) && root != emptyRoot { @@ -117,7 +120,7 @@ func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { // NewEmpty is a shortcut to create empty tree. It's mostly used in tests. func NewEmpty(db *Database) *Trie { - tr, _ := New(common.Hash{}, common.Hash{}, db) + tr, _ := New(common.Hash{}, common.Hash{}, common.Hash{}, db) return tr } @@ -219,7 +222,7 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new if hash == nil { return nil, origNode, 0, errors.New("non-consensus node") } - blob, err := t.db.Node(common.BytesToHash(hash)) + blob, err := t.reader.nodeBlob(t.owner, path, common.BytesToHash(hash)) return blob, origNode, 1, err } // Path still needs to be traversed, descend into children @@ -553,25 +556,16 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { return n, nil } -// resolveHash loads node from the underlying database with the provided -// node hash and path prefix. +// resolveHash loads node from the underlying store with the given node hash and +// path prefix. The rlp-encoded blob is preferred to be loaded from database +// because node pre-value is required to be tracked. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { - hash := common.BytesToHash(n) - if node := t.db.node(hash); node != nil { - return node, nil - } - return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} -} - -// resolveHash loads rlp-encoded node blob from the underlying database -// with the provided node hash and path prefix. -func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { - hash := common.BytesToHash(n) - blob, _ := t.db.Node(hash) - if len(blob) != 0 { - return blob, nil + blob, err := t.reader.nodeBlob(t.owner, prefix, common.BytesToHash(n)) + if err != nil { + return nil, err } - return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} + t.tracer.onRead(prefix, blob) + return mustDecodeNode(n, blob), nil } // Hash returns the root hash of the trie. It does not write to the @@ -606,7 +600,7 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { t.root = hashedNode return rootHash, nil, nil } - h := newCommitter(t.owner, collectLeaf) + h := newCommitter(t.owner, t.tracer, collectLeaf) newRoot, nodes, err := h.Commit(t.root) if err != nil { return common.Hash{}, nil, err @@ -633,6 +627,5 @@ func (t *Trie) Reset() { t.root = nil t.owner = common.Hash{} t.unhashed = 0 - //t.db = nil t.tracer.reset() } diff --git a/trie/trie_reader.go b/trie/trie_reader.go new file mode 100644 index 00000000000..0f883270bb5 --- /dev/null +++ b/trie/trie_reader.go @@ -0,0 +1,106 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" +) + +// Reader wraps the Node and NodeBlob method of a backing trie store. +type Reader interface { + // Node retrieves the trie node with the provided trie identifier, hexary + // node path and the corresponding node hash. + // No error will be returned if the node is not found. + Node(owner common.Hash, path []byte, hash common.Hash) (node, error) + + // NodeBlob retrieves the RLP-encoded trie node blob with the provided trie + // identifier, hexary node path and the corresponding node hash. + // No error will be returned if the node is not found. + NodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) +} + +// NodeReader warps all the necessary functions for accessing trie node. +type NodeReader interface { + // GetReader returns a reader for accessing all trie nodes with provided + // state root. Nil is returned in case the state is not available. + GetReader(root common.Hash) Reader +} + +// trieReader is a wrapper of the underlying node reader. It's not safe +// for concurrent usage. +type trieReader struct { + owner common.Hash + reader Reader + banned map[string]struct{} // Marker to prevent node from being accessed, for tests +} + +// newTrieReader initializes the trie reader with the given node reader. +func newTrieReader(owner common.Hash, root common.Hash, db NodeReader) (*trieReader, error) { + reader := db.GetReader(root) + if reader == nil { + return nil, fmt.Errorf("state not found #%x", root) + } + return &trieReader{owner: owner, reader: reader}, nil +} + +// newEmptyReader initializes the pure in-memory reader. All read operations +// should be forbidden and returns the MissingNodeError. +func newEmptyReader() *trieReader { + return &trieReader{} +} + +// node retrieves the trie node with the provided trie node information. +// An MissingNodeError will be returned in case the node is not found or +// any error is encountered. +func (r *trieReader) node(owner common.Hash, path []byte, hash common.Hash) (node, error) { + // Perform the logics in tests for preventing trie node access. + if r.banned != nil { + if _, ok := r.banned[string(path)]; ok { + return nil, &MissingNodeError{Owner: owner, NodeHash: hash, Path: path} + } + } + if r.reader == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + node, err := r.reader.Node(owner, path, hash) + if err != nil || node == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} + } + return node, nil +} + +// node retrieves the rlp-encoded trie node with the provided trie node +// information. An MissingNodeError will be returned in case the node is +// not found or any error is encountered. +func (r *trieReader) nodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + // Perform the logics in tests for preventing trie node access. + if r.banned != nil { + if _, ok := r.banned[string(path)]; ok { + return nil, &MissingNodeError{Owner: owner, NodeHash: hash, Path: path} + } + } + if r.reader == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + blob, err := r.reader.NodeBlob(owner, path, hash) + if err != nil || len(blob) == 0 { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} + } + return blob, nil +} diff --git a/trie/trie_test.go b/trie/trie_test.go index 3e29600bbd1..972a4845ada 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -64,7 +64,8 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.Hash{}, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) + root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + trie, err := New(root, common.Hash{}, root, NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -89,27 +90,27 @@ func testMissingNode(t *testing.T, memonly bool) { triedb.Commit(root, true, nil) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err := trie.TryGet([]byte("120000")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -122,27 +123,27 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb.Delete(hash[:]) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120000")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(common.Hash{}, root, triedb) + trie, _ = New(root, common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) @@ -196,7 +197,7 @@ func TestGet(t *testing.T) { } root, nodes, _ := trie.Commit(false) db.Update(NewWithNodeSet(nodes)) - trie, _ = New(common.Hash{}, root, db) + trie, _ = New(root, common.Hash{}, root, db) } } @@ -273,7 +274,7 @@ func TestReplication(t *testing.T) { triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(common.Hash{}, exp, triedb) + trie2, err := New(exp, common.Hash{}, exp, triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -294,7 +295,7 @@ func TestReplication(t *testing.T) { if nodes != nil { triedb.Update(NewWithNodeSet(nodes)) } - trie2, err = New(common.Hash{}, hash, triedb) + trie2, err = New(hash, common.Hash{}, hash, triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -439,21 +440,44 @@ func runRandTest(rt randTest) bool { case opHash: tr.Hash() case opCommit: - hash, nodes, err := tr.Commit(false) + root, nodes, err := tr.Commit(true) if err != nil { rt[i].err = err return false } + // Validity the returned nodeset + if nodes != nil { + for path, node := range nodes.updates.nodes { + blob, _, _ := origTrie.TryGetNode(hexToCompact([]byte(path))) + got := node.prev + if !bytes.Equal(blob, got) { + rt[i].err = fmt.Errorf("prevalue mismatch for 0x%x, got 0x%x want 0x%x", path, got, blob) + panic(rt[i].err) + return false + } + } + for path, prev := range nodes.deletes { + blob, _, _ := origTrie.TryGetNode(hexToCompact([]byte(path))) + if !bytes.Equal(blob, prev) { + rt[i].err = fmt.Errorf("prevalue mismatch for 0x%x, got 0x%x want 0x%x", path, prev, blob) + return false + } + } + } if nodes != nil { triedb.Update(NewWithNodeSet(nodes)) } - newtr, err := New(common.Hash{}, hash, triedb) + newtr, err := New(root, common.Hash{}, root, triedb) if err != nil { rt[i].err = err return false } tr = newtr + + // Enable node tracing. Resolve the root node again explicitly + // since it's not captured at the beginning. tr.tracer = newTracer() + tr.resolveHash(root.Bytes(), nil) origTrie = tr.Copy() case opItercheckhash: diff --git a/trie/util_test.go b/trie/util_test.go index cf6758e63d4..038e9faa573 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -72,7 +72,7 @@ func TestTrieTracer(t *testing.T) { if err := db.Update(NewWithNodeSet(nodes)); err != nil { t.Fatal(err) } - trie, _ = New(common.Hash{}, root, db) + trie, _ = New(root, common.Hash{}, root, db) trie.tracer = newTracer() // Delete all the elements, check deletion set diff --git a/trie/utils.go b/trie/utils.go index 7e26915041e..16a7d5c8639 100644 --- a/trie/utils.go +++ b/trie/utils.go @@ -50,45 +50,43 @@ func newTracer() *tracer { } } -/* // onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally. // Don't change the value outside of function since it's not deep-copied. -func (t *tracer) onRead(key []byte, val []byte) { +func (t *tracer) onRead(path []byte, val []byte) { // Tracer isn't used right now, remove this check later. if t == nil { return } - t.origin[string(key)] = val + t.origin[string(path)] = val } -*/ // onInsert tracks the newly inserted trie node. If it's already in the deletion set // (resurrected node), then just wipe it from the deletion set as the "untouched". -func (t *tracer) onInsert(key []byte) { +func (t *tracer) onInsert(path []byte) { // Tracer isn't used right now, remove this check later. if t == nil { return } - if _, present := t.delete[string(key)]; present { - delete(t.delete, string(key)) + if _, present := t.delete[string(path)]; present { + delete(t.delete, string(path)) return } - t.insert[string(key)] = struct{}{} + t.insert[string(path)] = struct{}{} } // onDelete tracks the newly deleted trie node. If it's already // in the addition set, then just wipe it from the addition set // as it's untouched. -func (t *tracer) onDelete(key []byte) { +func (t *tracer) onDelete(path []byte) { // Tracer isn't used right now, remove this check later. if t == nil { return } - if _, present := t.insert[string(key)]; present { - delete(t.insert, string(key)) + if _, present := t.insert[string(path)]; present { + delete(t.insert, string(path)) return } - t.delete[string(key)] = struct{}{} + t.delete[string(path)] = struct{}{} } // insertList returns the tracked inserted trie nodes in list format. @@ -98,8 +96,8 @@ func (t *tracer) insertList() [][]byte { return nil } var ret [][]byte - for key := range t.insert { - ret = append(ret, []byte(key)) + for path := range t.insert { + ret = append(ret, []byte(path)) } return ret } @@ -111,22 +109,20 @@ func (t *tracer) deleteList() [][]byte { return nil } var ret [][]byte - for key := range t.delete { - ret = append(ret, []byte(key)) + for path := range t.delete { + ret = append(ret, []byte(path)) } return ret } -/* // getPrev returns the cached original value of the specified node. -func (t *tracer) getPrev(key []byte) []byte { - // Don't panic on uninitialized tracer, it's possible in testing. +func (t *tracer) getPrev(path []byte) []byte { + // Tracer isn't used right now, remove this check later. if t == nil { return nil } - return t.origin[string(key)] + return t.origin[string(path)] } -*/ // reset clears the content tracked by tracer. func (t *tracer) reset() { From a91e35c5d1b296cb2b6bde5ea616e25e8d766460 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 20 Sep 2022 10:27:58 +0800 Subject: [PATCH 02/10] trie: add docs --- trie/trie.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/trie/trie.go b/trie/trie.go index c29b4f40e1b..dcddfadb791 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -91,13 +91,19 @@ func (t *Trie) Copy() *Trie { } } -// New creates a trie with an existing root node from db and an assigned -// owner for storage proximity. +// New creates the trie instance with provided information and read-only database. // -// If root is the zero hash or the sha3 hash of an empty string, the -// trie is initially empty and does not require a database. Otherwise, -// New will panic if db is nil and returns a MissingNodeError if root does -// not exist in the database. Accessing the trie loads nodes from db on demand. +// - stateRoot: the identifier for selecting state to access trie nodes. +// If the corresponding state is not available, an error will be returned. +// +// - owner: the identifier of second-layer trie(namely the storage trie in +// ethereum). It's the hash of the corresponding contract address +// used for uniquely identifying trie nodes. It's empty for account trie. +// +// - root: the root hash of the trie. If root is the zero hash or the sha3 +// hash of an empty string, then trie is initially empty. Otherwise, +// the root node must be present in database or returns a MissingNodeError +// if not. func New(stateRoot common.Hash, owner common.Hash, root common.Hash, db NodeReader) (*Trie, error) { reader, err := newTrieReader(owner, stateRoot, db) if err != nil { From 90a63c59e38da490efd227a3b81e0d553aaf1d1d Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 22 Sep 2022 10:06:46 +0800 Subject: [PATCH 03/10] trie: address comments --- trie/committer.go | 2 +- trie/trie_reader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/committer.go b/trie/committer.go index b241b34eecf..90191cf9b1d 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -62,7 +62,7 @@ func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { // embedded in the parent and now deleted from the trie. In this case // it's noop from database's perspective. val := c.tracer.getPrev(path) - if val == nil { + if len(val) == 0 { continue } c.nodes.markDeleted(path, val) diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 0f883270bb5..c235da2ea9f 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -35,7 +35,7 @@ type Reader interface { NodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) } -// NodeReader warps all the necessary functions for accessing trie node. +// NodeReader wraps all the necessary functions for accessing trie node. type NodeReader interface { // GetReader returns a reader for accessing all trie nodes with provided // state root. Nil is returned in case the state is not available. From 1f72e4569025942f4df784011e4087251d8afda2 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 22 Sep 2022 19:44:44 +0800 Subject: [PATCH 04/10] cmd, core, eth, les, light, trie: trie id --- cmd/geth/dbcmd.go | 27 +++++++++----- cmd/geth/snapshot.go | 10 +++-- core/state/database.go | 4 +- core/state/pruner/pruner.go | 5 ++- core/state/snapshot/generate.go | 23 +++++++----- core/state/snapshot/generate_test.go | 5 ++- core/state/sync_test.go | 7 ++-- eth/api.go | 4 +- eth/protocols/snap/handler.go | 12 +++--- eth/protocols/snap/sync_test.go | 23 +++++++----- les/downloader/downloader_test.go | 2 +- les/handler_test.go | 8 ++-- les/request_test.go | 1 + les/server_handler.go | 4 +- light/postprocess.go | 12 +++--- light/trie.go | 16 +++++--- tests/fuzzers/trie/trie-fuzzer.go | 3 +- trie/iterator.go | 4 +- trie/iterator_test.go | 16 ++++---- trie/secure_trie.go | 11 ++++-- trie/secure_trie_test.go | 6 +-- trie/sync_test.go | 12 +++--- trie/trie.go | 35 +++++++----------- trie/trie_id.go | 55 ++++++++++++++++++++++++++++ trie/trie_reader.go | 18 ++++----- trie/trie_test.go | 30 +++++++-------- trie/util_test.go | 3 +- 27 files changed, 216 insertions(+), 140 deletions(-) create mode 100644 trie/trie_id.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 8f24ef91ee5..9d834ee14b9 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -150,7 +150,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Action: dbDumpTrie, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", - ArgsUsage: " ", + ArgsUsage: " ", Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -486,7 +486,7 @@ func dbPut(ctx *cli.Context) error { // dbDumpTrie shows the key-value slots of a given storage trie func dbDumpTrie(ctx *cli.Context) error { - if ctx.NArg() < 2 { + if ctx.NArg() < 3 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) @@ -494,34 +494,41 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() + var ( state []byte storage []byte + account []byte start []byte max = int64(-1) err error ) if state, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { - log.Info("Could not decode the root", "error", err) + log.Info("Could not decode the state root", "error", err) + return err + } + if account, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + log.Info("Could not decode the account hash", "error", err) return err } - if storage, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { - log.Info("Could not decode the root", "error", err) + if storage, err = hexutil.Decode(ctx.Args().Get(2)); err != nil { + log.Info("Could not decode the storage trie root", "error", err) return err } - if ctx.NArg() >= 3 { - if start, err = hexutil.Decode(ctx.Args().Get(2)); err != nil { + if ctx.NArg() > 3 { + if start, err = hexutil.Decode(ctx.Args().Get(3)); err != nil { log.Info("Could not decode the seek position", "error", err) return err } } - if ctx.NArg() >= 4 { - if max, err = strconv.ParseInt(ctx.Args().Get(3), 10, 64); err != nil { + if ctx.NArg() > 4 { + if max, err = strconv.ParseInt(ctx.Args().Get(4), 10, 64); err != nil { log.Info("Could not decode the max count", "error", err) return err } } - theTrie, err := trie.New(common.BytesToHash(state), common.Hash{}, common.BytesToHash(storage), trie.NewDatabase(db)) + id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) + theTrie, err := trie.New(id, trie.NewDatabase(db)) if err != nil { return err } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 637c6fa6863..aade9254177 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -271,7 +271,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewStateTrie(root, common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -292,7 +292,8 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIter.Key), acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root) + storageTrie, err := trie.NewStateTrie(id, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -360,7 +361,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewStateTrie(root, common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -406,7 +407,8 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root) + storageTrie, err := trie.NewStateTrie(id, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") diff --git a/core/state/database.go b/core/state/database.go index 1aa901216b7..5e3d9a9d388 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -148,7 +148,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewStateTrie(root, common.Hash{}, root, db.db) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.db) if err != nil { return nil, err } @@ -157,7 +157,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewStateTrie(stateRoot, addrHash, root, db.db) + tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, addrHash, root), db.db) if err != nil { return nil, err } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 46b9481b55e..b2f815d9f75 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(genesis.Root(), common.Hash{}, genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,8 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewStateTrie(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) + id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) + storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 310931e4b6e..2988d582ffd 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error) // // The proof result will be returned if the range proving is finished, otherwise // the error will be returned to abort the entire procedure. -func (dl *diskLayer) proveRange(ctx *generatorContext, stateRoot common.Hash, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { +func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { var ( keys [][]byte vals [][]byte @@ -233,8 +233,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, stateRoot common.Hash, ow }(time.Now()) // The snap state is exhausted, pass the entire key/val set for verification + root := trieId.Root if origin == nil && !diskMore { - stackTr := trie.NewStackTrieWithOwner(nil, owner) + stackTr := trie.NewStackTrieWithOwner(nil, trieId.Owner) for i, key := range keys { stackTr.TryUpdate(key, vals[i]) } @@ -248,7 +249,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, stateRoot common.Hash, ow return &proofResult{keys: keys, vals: vals}, nil } // Snap state is chunked, generate edge proofs for verification. - tr, err := trie.New(stateRoot, owner, root, dl.triedb) + tr, err := trie.New(trieId, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie @@ -313,9 +314,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error // generateRange generates the state segment with particular prefix. Generation can // either verify the correctness of existing state through range-proof and skip // generation, or iterate trie to regenerate state on demand. -func (dl *diskLayer) generateRange(ctx *generatorContext, state common.Hash, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { +func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { // Use range prover to check the validity of the flat state in the range - result, err := dl.proveRange(ctx, state, owner, root, prefix, kind, origin, max, valueConvertFn) + result, err := dl.proveRange(ctx, trieId, prefix, kind, origin, max, valueConvertFn) if err != nil { return false, nil, err } @@ -363,7 +364,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, state common.Hash, own if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(common.Hash{}, owner, common.Hash{}, snapTrieDb) + snapTrie, _ := trie.New(trie.TrieID(common.Hash{}), snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } @@ -377,7 +378,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, state common.Hash, own // if it's already opened with some nodes resolved. tr := result.tr if tr == nil { - tr, err = trie.New(state, owner, root, dl.triedb) + tr, err = trie.New(trieId, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return false, nil, errMissingTrie @@ -460,7 +461,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, state common.Hash, own } else { snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) } - logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last), + logger.Debug("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last), "count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted) // If there are either more trie items, or there are more snap items @@ -540,7 +541,8 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Has // Loop for re-generating the missing storage slots. var origin = common.CopyBytes(storeMarker) for { - exhausted, last, err := dl.generateRange(ctx, stateRoot, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + id := trie.StorageTrieID(stateRoot, account, storageRoot) + exhausted, last, err := dl.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } @@ -640,7 +642,8 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er } origin := common.CopyBytes(accMarker) for { - exhausted, last, err := dl.generateRange(ctx, dl.root, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + id := trie.StateTrieID(dl.root) + exhausted, last, err := dl.generateRange(ctx, id, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 8c6ac4d0ca2..784d76859e4 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -149,7 +149,7 @@ type testHelper struct { func newHelper() *testHelper { diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(trie.StateTrieID(common.Hash{}), triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, @@ -182,7 +182,8 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { - stTrie, _ := trie.NewStateTrie(stateRoot, owner, common.Hash{}, t.triedb) + id := trie.StorageTrieID(stateRoot, owner, common.Hash{}) + stTrie, _ := trie.NewStateTrie(id, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 1e73c54ab8a..dbcbb7c9634 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.KeyValueStore, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(root, common.Hash{}, root, trie.NewDatabase(db)) + trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db)) if err != nil { return err } @@ -174,7 +174,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if commit { srcDb.TrieDB().Commit(srcRoot, false, nil) } - srcTrie, _ := trie.New(srcRoot, common.Hash{}, srcRoot, srcDb.TrieDB()) + srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -222,7 +222,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) } - stTrie, err := trie.New(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB()) + id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) + stTrie, err := trie.New(id, srcDb.TrieDB()) if err != nil { t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) } diff --git a/eth/api.go b/eth/api.go index 556ee10c363..e480dde8f64 100644 --- a/eth/api.go +++ b/eth/api.go @@ -508,11 +508,11 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewStateTrie(startBlock.Root(), common.Hash{}, startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewStateTrie(endBlock.Root(), common.Hash{}, endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(trie.StateTrieID(endBlock.Root()), triedb) if err != nil { return nil, err } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index c35cd68d358..aa245ab7e62 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -283,7 +283,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(req.Root, common.Hash{}, req.Root, chain.StateCache().TrieDB()) + tr, err := trie.New(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -413,7 +413,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.NewStateTrie(req.Root, common.Hash{}, req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -421,7 +421,8 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.NewStateTrie(req.Root, account, acc.Root, chain.StateCache().TrieDB()) + id := trie.StorageTrieID(req.Root, account, acc.Root) + stTrie, err := trie.NewStateTrie(id, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -487,7 +488,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewStateTrie(req.Root, common.Hash{}, req.Root, triedb) + accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -529,7 +530,8 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewStateTrie(req.Root, common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) + id := trie.StorageTrieID(req.Root, common.BytesToHash(pathset[0]), common.BytesToHash(account.Root)) + stTrie, err := trie.NewStateTrie(id, triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index ee6288e4b53..1d1ce932e07 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1372,7 +1372,7 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { root, nodes, _ := accTrie.Commit(false) db.Update(trie.NewWithNodeSet(nodes)) - accTrie, _ = trie.New(root, common.Hash{}, root, db) + accTrie, _ = trie.New(trie.StateTrieID(root), db) return accTrie, entries } @@ -1434,7 +1434,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { root, nodes, _ := accTrie.Commit(false) db.Update(trie.NewWithNodeSet(nodes)) - accTrie, _ = trie.New(root, common.Hash{}, root, db) + accTrie, _ = trie.New(trie.StateTrieID(root), db) return accTrie, entries } @@ -1484,10 +1484,11 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) db.Update(nodes) // Re-create tries with new root - accTrie, _ = trie.New(root, common.Hash{}, root, db) + accTrie, _ = trie.New(trie.StateTrieID(root), db) for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) - trie, _ := trie.New(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + id := trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]) + trie, _ := trie.New(id, db) storageTries[common.BytesToHash(key)] = trie } return accTrie, entries, storageTries, storageEntries @@ -1548,13 +1549,14 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie db.Update(nodes) // Re-create tries with new root - accTrie, err := trie.New(root, common.Hash{}, root, db) + accTrie, err := trie.New(trie.StateTrieID(root), db) if err != nil { panic(err) } for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) - trie, err := trie.New(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + id := trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]) + trie, err := trie.New(id, db) if err != nil { panic(err) } @@ -1567,7 +1569,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { - trie, _ := trie.New(common.Hash{}, owner, common.Hash{}, db) + trie, _ := trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1593,7 +1595,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(common.Hash{}, owner, common.Hash{}, db) + trie, _ = trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) ) // Initialize boundaries var next common.Hash @@ -1640,7 +1642,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(db) - accTrie, err := trie.New(root, common.Hash{}, root, triedb) + accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) } @@ -1658,7 +1660,8 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewStateTrie(root, common.BytesToHash(accIt.Key), acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIt.Key), acc.Root) + storeTrie, err := trie.NewStateTrie(id, triedb) if err != nil { t.Fatal(err) } diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index 9a9aa0a8131..1704d3e7433 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewStateTrie(block.Root(), common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/les/handler_test.go b/les/handler_test.go index cf66184247e..ecf97bf9d10 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(header.Root, common.Hash{}, header.Root, trie.NewDatabase(server.db)) + trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(header.Root, common.Hash{}, header.Root, trie.NewDatabase(server.db)) + t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/request_test.go b/les/request_test.go index c65405e3752..9b52e6bd86a 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -104,6 +104,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { bhash := rawdb.ReadCanonicalHash(server.db, i) if req := fn(client.db, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + err := client.handler.backend.odr.Retrieve(ctx, req) cancel() diff --git a/les/server_handler.go b/les/server_handler.go index b58926009c4..32a38f64cc4 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool { // getAccount retrieves an account from the state based on root. func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(root, common.Hash{}, root, triedb) + trie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { return types.StateAccount{}, err } @@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(root, common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) return trie } diff --git a/light/postprocess.go b/light/postprocess.go index f19c1e8861d..bd17eca8a3d 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -184,12 +184,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error - c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) + c.trie, err = trie.New(trie.TrieID(root), c.triedb) if err != nil && c.odr != nil { err = c.fetchMissingNodes(ctx, section, root) if err == nil { - c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) + c.trie, err = trie.New(trie.TrieID(root), c.triedb) } } c.section = section @@ -228,7 +228,7 @@ func (c *ChtIndexerBackend) Commit() error { } } // Re-create trie with newly generated root and updated database. - c.trie, err = trie.New(root, common.Hash{}, root, c.triedb) + c.trie, err = trie.New(trie.TrieID(root), c.triedb) if err != nil { return err } @@ -415,11 +415,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error - b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) + b.trie, err = trie.New(trie.TrieID(root), b.triedb) if err != nil && b.odr != nil { err = b.fetchMissingNodes(ctx, section, root) if err == nil { - b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) + b.trie, err = trie.New(trie.TrieID(root), b.triedb) } } b.section = section @@ -479,7 +479,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { } } // Re-create trie with newly generated root and updated database. - b.trie, err = trie.New(root, common.Hash{}, root, b.triedb) + b.trie, err = trie.New(trie.TrieID(root), b.triedb) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 12beed97cff..0092eee136c 100644 --- a/light/trie.go +++ b/light/trie.go @@ -196,11 +196,13 @@ func (t *odrTrie) do(key []byte, fn func() error) error { for { var err error if t.trie == nil { - var owner common.Hash + var id *trie.ID if len(t.id.AccKey) > 0 { - owner = common.BytesToHash(t.id.AccKey) + id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + } else { + id = trie.StateTrieID(t.id.StateRoot) } - t.trie, err = trie.New(t.id.StateRoot, owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) + t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database())) } if err == nil { err = fn() @@ -226,11 +228,13 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { // Open the actual non-ODR trie if that hasn't happened yet. if t.trie == nil { it.do(func() error { - var owner common.Hash + var id *trie.ID if len(t.id.AccKey) > 0 { - owner = common.BytesToHash(t.id.AccKey) + id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + } else { + id = trie.StateTrieID(t.id.StateRoot) } - t, err := trie.New(t.id.StateRoot, owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) + t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database())) if err == nil { it.t.trie = t } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 52b8d9f1dc6..8467bdafa6b 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -21,7 +21,6 @@ import ( "encoding/binary" "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie" ) @@ -170,7 +169,7 @@ func runRandTest(rt randTest) error { return err } } - newtr, err := trie.New(hash, common.Hash{}, hash, triedb) + newtr, err := trie.New(trie.TrieID(hash), triedb) if err != nil { return err } diff --git a/trie/iterator.go b/trie/iterator.go index 547a0d3a829..724a7aaf11e 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -380,7 +380,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { // loaded blob will be tracked, while it's not required here since // all loaded nodes won't be linked to trie at all and track nodes // may lead to out-of-memory issue. - return it.trie.reader.node(it.trie.owner, path, common.BytesToHash(hash)) + return it.trie.reader.node(path, common.BytesToHash(hash)) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { @@ -394,7 +394,7 @@ func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) // loaded blob will be tracked, while it's not required here since // all loaded nodes won't be linked to trie at all and track nodes // may lead to out-of-memory issue. - return it.trie.reader.nodeBlob(it.trie.owner, path, common.BytesToHash(hash)) + return it.trie.reader.nodeBlob(path, common.BytesToHash(hash)) } func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 46edbe0104a..74b87a25c23 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -66,7 +66,7 @@ func TestIterator(t *testing.T) { } db.Update(NewWithNodeSet(nodes)) - trie, _ = New(root, common.Hash{}, root, db) + trie, _ = New(TrieID(root), db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -227,7 +227,7 @@ func TestDifferenceIterator(t *testing.T) { } rootA, nodesA, _ := triea.Commit(false) dba.Update(NewWithNodeSet(nodesA)) - triea, _ = New(rootA, common.Hash{}, rootA, dba) + triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) trieb := NewEmpty(dbb) @@ -236,7 +236,7 @@ func TestDifferenceIterator(t *testing.T) { } rootB, nodesB, _ := trieb.Commit(false) dbb.Update(NewWithNodeSet(nodesB)) - trieb, _ = New(rootB, common.Hash{}, rootB, dbb) + trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -269,7 +269,7 @@ func TestUnionIterator(t *testing.T) { } rootA, nodesA, _ := triea.Commit(false) dba.Update(NewWithNodeSet(nodesA)) - triea, _ = New(rootA, common.Hash{}, rootA, dba) + triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) trieb := NewEmpty(dbb) @@ -278,7 +278,7 @@ func TestUnionIterator(t *testing.T) { } rootB, nodesB, _ := trieb.Commit(false) dbb.Update(NewWithNodeSet(nodesB)) - trieb, _ = New(rootB, common.Hash{}, rootB, dbb) + trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -356,7 +356,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. - tr, _ := New(tr.Hash(), common.Hash{}, tr.Hash(), triedb) + tr, _ := New(TrieID(tr.Hash()), triedb) // Remove a random node from the database. It can't be the root node // because that one is already loaded. @@ -445,7 +445,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - tr, _ := New(root, common.Hash{}, root, triedb) + tr, _ := New(TrieID(root), triedb) it := tr.NodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { @@ -533,7 +533,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 3f49480722d..96faab15826 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -30,7 +30,12 @@ type SecureTrie = StateTrie // NewSecure creates a new StateTrie. // Deprecated: use NewStateTrie. func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { - return NewStateTrie(stateRoot, owner, root, db) + id := &ID{ + StateRoot: stateRoot, + Owner: owner, + Root: root, + } + return NewStateTrie(id, db) } // StateTrie wraps a trie with key hashing. In a stateTrie trie, all @@ -56,11 +61,11 @@ type StateTrie struct { // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil // and returns MissingNodeError if the root node cannot be found. -func NewStateTrie(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { +func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { if db == nil { panic("trie.NewStateTrie called without a database") } - trie, err := New(stateRoot, owner, root, db) + trie, err := New(id, db) if err != nil { return nil, err } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 6b4839a5d7c..ab8462607d9 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -29,7 +29,7 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) + trie, _ := NewStateTrie(TrieID(common.Hash{}), NewDatabase(memorydb.New())) return trie } @@ -37,7 +37,7 @@ func newEmptySecure() *StateTrie { func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -66,7 +66,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state - trie, _ = NewStateTrie(root, common.Hash{}, root, triedb) + trie, _ = NewStateTrie(TrieID(root), triedb) return triedb, trie, content } diff --git a/trie/sync_test.go b/trie/sync_test.go index 8b52bd352dc..a0252785530 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -30,7 +30,7 @@ import ( func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -59,7 +59,7 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state - trie, _ = NewStateTrie(root, common.Hash{}, root, triedb) + trie, _ = NewStateTrie(TrieID(root), triedb) return triedb, trie, content } @@ -67,7 +67,7 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewStateTrie(common.BytesToHash(root), common.Hash{}, common.BytesToHash(root), db) + trie, err := NewStateTrie(TrieID(common.BytesToHash(root)), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -84,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewStateTrie(root, common.Hash{}, root, db) + trie, err := NewStateTrie(TrieID(root), db) if err != nil { return nil // Consider a non existent state consistent } @@ -105,8 +105,8 @@ type trieElement struct { func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New()) - emptyA := NewEmpty(dbA) - emptyB, _ := New(emptyRoot, common.Hash{}, emptyRoot, dbB) + emptyA, _ := New(TrieID(common.Hash{}), dbA) + emptyB, _ := New(TrieID(emptyRoot), dbB) for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) diff --git a/trie/trie.go b/trie/trie.go index dcddfadb791..64014578e5d 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -91,31 +91,24 @@ func (t *Trie) Copy() *Trie { } } -// New creates the trie instance with provided information and read-only database. -// -// - stateRoot: the identifier for selecting state to access trie nodes. -// If the corresponding state is not available, an error will be returned. -// -// - owner: the identifier of second-layer trie(namely the storage trie in -// ethereum). It's the hash of the corresponding contract address -// used for uniquely identifying trie nodes. It's empty for account trie. -// -// - root: the root hash of the trie. If root is the zero hash or the sha3 -// hash of an empty string, then trie is initially empty. Otherwise, -// the root node must be present in database or returns a MissingNodeError -// if not. -func New(stateRoot common.Hash, owner common.Hash, root common.Hash, db NodeReader) (*Trie, error) { - reader, err := newTrieReader(owner, stateRoot, db) +// New creates the trie instance with provided trie id and the read-only +// database. The state specified by trie id must be available, otherwise +// an error will be returned. The trie root specified by trie id can be +// zero hash or the sha3 hash of an empty string, then trie is initially +// empty, otherwise, the root node must be present in database or returns +// a MissingNodeError if not. +func New(id *ID, db NodeReader) (*Trie, error) { + reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { return nil, err } trie := &Trie{ - owner: owner, + owner: id.Owner, reader: reader, //tracer: newTracer(), } - if root != (common.Hash{}) && root != emptyRoot { - rootnode, err := trie.resolveHash(root[:], nil) + if id.Root != (common.Hash{}) && id.Root != emptyRoot { + rootnode, err := trie.resolveHash(id.Root[:], nil) if err != nil { return nil, err } @@ -126,7 +119,7 @@ func New(stateRoot common.Hash, owner common.Hash, root common.Hash, db NodeRead // NewEmpty is a shortcut to create empty tree. It's mostly used in tests. func NewEmpty(db *Database) *Trie { - tr, _ := New(common.Hash{}, common.Hash{}, common.Hash{}, db) + tr, _ := New(TrieID(common.Hash{}), db) return tr } @@ -228,7 +221,7 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new if hash == nil { return nil, origNode, 0, errors.New("non-consensus node") } - blob, err := t.reader.nodeBlob(t.owner, path, common.BytesToHash(hash)) + blob, err := t.reader.nodeBlob(path, common.BytesToHash(hash)) return blob, origNode, 1, err } // Path still needs to be traversed, descend into children @@ -566,7 +559,7 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { // path prefix. The rlp-encoded blob is preferred to be loaded from database // because node pre-value is required to be tracked. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { - blob, err := t.reader.nodeBlob(t.owner, prefix, common.BytesToHash(n)) + blob, err := t.reader.nodeBlob(prefix, common.BytesToHash(n)) if err != nil { return nil, err } diff --git a/trie/trie_id.go b/trie/trie_id.go new file mode 100644 index 00000000000..8ab490ca3b1 --- /dev/null +++ b/trie/trie_id.go @@ -0,0 +1,55 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package trie + +import "github.com/ethereum/go-ethereum/common" + +// ID is the identifier for uniquely identifying a trie. +type ID struct { + StateRoot common.Hash // The root of the corresponding state(block.root) + Owner common.Hash // The contract address hash which the trie belongs to + Root common.Hash // The root hash of trie +} + +// StateTrieID constructs an identifier for state trie with the provided state root. +func StateTrieID(root common.Hash) *ID { + return &ID{ + StateRoot: root, + Owner: common.Hash{}, + Root: root, + } +} + +// StorageTrieID constructs an identifier for storage trie which belongs to a certain +// state and contract specified by the stateRoot and owner. +func StorageTrieID(stateRoot common.Hash, owner common.Hash, root common.Hash) *ID { + return &ID{ + StateRoot: stateRoot, + Owner: owner, + Root: root, + } +} + +// TrieID constructs an identifier for a standard trie(not a second-layer trie) +// with provided root. It's mostly used in tests and some other tries like CHT trie. +func TrieID(root common.Hash) *ID { + return &ID{ + StateRoot: root, + Owner: common.Hash{}, + Root: root, + } +} diff --git a/trie/trie_reader.go b/trie/trie_reader.go index c235da2ea9f..14186159b71 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -51,10 +51,10 @@ type trieReader struct { } // newTrieReader initializes the trie reader with the given node reader. -func newTrieReader(owner common.Hash, root common.Hash, db NodeReader) (*trieReader, error) { - reader := db.GetReader(root) +func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, error) { + reader := db.GetReader(stateRoot) if reader == nil { - return nil, fmt.Errorf("state not found #%x", root) + return nil, fmt.Errorf("state not found #%x", stateRoot) } return &trieReader{owner: owner, reader: reader}, nil } @@ -68,17 +68,17 @@ func newEmptyReader() *trieReader { // node retrieves the trie node with the provided trie node information. // An MissingNodeError will be returned in case the node is not found or // any error is encountered. -func (r *trieReader) node(owner common.Hash, path []byte, hash common.Hash) (node, error) { +func (r *trieReader) node(path []byte, hash common.Hash) (node, error) { // Perform the logics in tests for preventing trie node access. if r.banned != nil { if _, ok := r.banned[string(path)]; ok { - return nil, &MissingNodeError{Owner: owner, NodeHash: hash, Path: path} + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } } if r.reader == nil { return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } - node, err := r.reader.Node(owner, path, hash) + node, err := r.reader.Node(r.owner, path, hash) if err != nil || node == nil { return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} } @@ -88,17 +88,17 @@ func (r *trieReader) node(owner common.Hash, path []byte, hash common.Hash) (nod // node retrieves the rlp-encoded trie node with the provided trie node // information. An MissingNodeError will be returned in case the node is // not found or any error is encountered. -func (r *trieReader) nodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { +func (r *trieReader) nodeBlob(path []byte, hash common.Hash) ([]byte, error) { // Perform the logics in tests for preventing trie node access. if r.banned != nil { if _, ok := r.banned[string(path)]; ok { - return nil, &MissingNodeError{Owner: owner, NodeHash: hash, Path: path} + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } } if r.reader == nil { return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} } - blob, err := r.reader.NodeBlob(owner, path, hash) + blob, err := r.reader.NodeBlob(r.owner, path, hash) if err != nil || len(blob) == 0 { return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} } diff --git a/trie/trie_test.go b/trie/trie_test.go index 972a4845ada..0d37d213ab1 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -65,7 +65,7 @@ func TestNull(t *testing.T) { func TestMissingRoot(t *testing.T) { root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") - trie, err := New(root, common.Hash{}, root, NewDatabase(memorydb.New())) + trie, err := New(TrieID(root), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -90,27 +90,27 @@ func testMissingNode(t *testing.T, memonly bool) { triedb.Commit(root, true, nil) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err := trie.TryGet([]byte("120000")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120099")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryDelete([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -123,27 +123,27 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb.Delete(hash[:]) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120000")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120099")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, common.Hash{}, root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryDelete([]byte("123456")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) @@ -197,7 +197,7 @@ func TestGet(t *testing.T) { } root, nodes, _ := trie.Commit(false) db.Update(NewWithNodeSet(nodes)) - trie, _ = New(root, common.Hash{}, root, db) + trie, _ = New(TrieID(root), db) } } @@ -274,7 +274,7 @@ func TestReplication(t *testing.T) { triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(exp, common.Hash{}, exp, triedb) + trie2, err := New(TrieID(exp), triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -295,7 +295,7 @@ func TestReplication(t *testing.T) { if nodes != nil { triedb.Update(NewWithNodeSet(nodes)) } - trie2, err = New(hash, common.Hash{}, hash, triedb) + trie2, err = New(TrieID(hash), triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -467,7 +467,7 @@ func runRandTest(rt randTest) bool { if nodes != nil { triedb.Update(NewWithNodeSet(nodes)) } - newtr, err := New(root, common.Hash{}, root, triedb) + newtr, err := New(TrieID(root), triedb) if err != nil { rt[i].err = err return false diff --git a/trie/util_test.go b/trie/util_test.go index 038e9faa573..8c7ece03f6c 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -19,7 +19,6 @@ package trie import ( "testing" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) @@ -72,7 +71,7 @@ func TestTrieTracer(t *testing.T) { if err := db.Update(NewWithNodeSet(nodes)); err != nil { t.Fatal(err) } - trie, _ = New(root, common.Hash{}, root, db) + trie, _ = New(TrieID(root), db) trie.tracer = newTracer() // Delete all the elements, check deletion set From 8c34dacac871cdde5cd8372bd5e975cae273137c Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 26 Sep 2022 13:32:25 +0800 Subject: [PATCH 05/10] trie: add tests --- trie/proof.go | 2 +- trie/trie_test.go | 8 +++- trie/util_test.go | 119 ++++++++++++++++++++++++++++++++++++++++++++++ trie/utils.go | 17 +++++++ 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index 11ad44fc8d5..a95b60967ea 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -60,7 +60,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e nodes = append(nodes, n) case hashNode: var err error - tn, err = t.resolveHash(n, prefix) + tn, err = t.reader.node(prefix, common.BytesToHash(n)) if err != nil { log.Error("Unhandled trie error in Trie.Prove", "err", err) return err diff --git a/trie/trie_test.go b/trie/trie_test.go index 0d37d213ab1..1f3c8f3925f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -378,6 +378,7 @@ const ( opCommit opItercheckhash opNodeDiff + opProve opMax // boundary value, not an actual op ) @@ -403,7 +404,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { step.key = genKey() step.value = make([]byte, 8) binary.BigEndian.PutUint64(step.value, uint64(i)) - case opGet, opDelete: + case opGet, opDelete, opProve: step.key = genKey() } steps = append(steps, step) @@ -437,6 +438,11 @@ func runRandTest(rt randTest) bool { if string(v) != want { rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } + case opProve: + err := tr.Prove(step.key, 0, rawdb.NewMemoryDatabase()) + if err != nil { + rt[i].err = fmt.Errorf("failed for proving key %#x, %v", step.key, err) + } case opHash: tr.Hash() case opCommit: diff --git a/trie/util_test.go b/trie/util_test.go index 8c7ece03f6c..c9629899402 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -17,6 +17,8 @@ package trie import ( + "bytes" + "github.com/ethereum/go-ethereum/common" "testing" "github.com/ethereum/go-ethereum/core/rawdb" @@ -123,3 +125,120 @@ func TestTrieTracerNoop(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(trie.tracer.deleteList())) } } + +func TestTrieTracePrevValue(t *testing.T) { + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) + trie.tracer = newTracer() + + paths, blobs := trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + // Insert a batch of entries, all the nodes should be marked as inserted + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"dog", "puppy"}, + {"somethingveryoddindeedthis is", "myothernodedata"}, + } + for _, val := range vals { + trie.Update([]byte(val.k), []byte(val.v)) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + if err := db.Update(NewWithNodeSet(nodes)); err != nil { + t.Fatal(err) + } + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + trie.resolveHash(root.Bytes(), nil) + + // Load all nodes in trie + for _, val := range vals { + trie.TryGet([]byte(val.k)) + } + + // Ensure all nodes are tracked by tracer with correct prev-values + iter := trie.NodeIterator(nil) + seen := make(map[string][]byte) + for iter.Next(true) { + // Embedded nodes are ignored since they are not present in + // database. + if iter.Hash() == (common.Hash{}) { + continue + } + seen[string(iter.Path())] = common.CopyBytes(iter.NodeBlob()) + } + + paths, blobs = trie.tracer.prevList() + if len(paths) != len(seen) || len(blobs) != len(seen) { + t.Fatalf("Unexpected tracked values") + } + for i, path := range paths { + blob := blobs[i] + prev, ok := seen[string(path)] + if !ok { + t.Fatalf("Missing node %v", path) + } + if !bytes.Equal(blob, prev) { + t.Fatalf("Unexpected value path: %v, want: %v, got: %v", path, prev, blob) + } + } + + // Re-open the trie and iterate the trie, ensure nothing will be tracked. + // Iterator will not link any loaded nodes to trie. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + + iter = trie.NodeIterator(nil) + for iter.Next(true) { + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Re-open the trie and generate proof for entries, ensure nothing will + // be tracked. Prover will not link any loaded nodes to trie. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + for _, val := range vals { + trie.Prove([]byte(val.k), 0, rawdb.NewMemoryDatabase()) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Delete entries from trie, ensure all previous values are correct. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + trie.resolveHash(root.Bytes(), nil) + + for _, val := range vals { + trie.TryDelete([]byte(val.k)) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != len(seen) || len(blobs) != len(seen) { + t.Fatalf("Unexpected tracked values") + } + for i, path := range paths { + blob := blobs[i] + prev, ok := seen[string(path)] + if !ok { + t.Fatalf("Missing node %v", path) + } + if !bytes.Equal(blob, prev) { + t.Fatalf("Unexpected value path: %v, want: %v, got: %v", path, prev, blob) + } + } +} diff --git a/trie/utils.go b/trie/utils.go index 16a7d5c8639..d462b31bd20 100644 --- a/trie/utils.go +++ b/trie/utils.go @@ -115,6 +115,23 @@ func (t *tracer) deleteList() [][]byte { return ret } +// prevList returns the tracked node blobs in list format. +func (t *tracer) prevList() ([][]byte, [][]byte) { + // Tracer isn't used right now, remove this check later. + if t == nil { + return nil, nil + } + var ( + paths [][]byte + blobs [][]byte + ) + for path, blob := range t.origin { + paths = append(paths, []byte(path)) + blobs = append(blobs, blob) + } + return paths, blobs +} + // getPrev returns the cached original value of the specified node. func (t *tracer) getPrev(path []byte) []byte { // Tracer isn't used right now, remove this check later. From f1393bffc9dc176167ef556803b66378692de64b Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 26 Sep 2022 14:10:28 +0800 Subject: [PATCH 06/10] trie, core: updates --- core/state/snapshot/generate.go | 4 ++-- trie/iterator.go | 4 ++-- trie/proof.go | 5 +++++ trie/trie.go | 21 +++++++++++---------- trie/trie_test.go | 13 +++++++++++-- trie/util_test.go | 4 ++-- 6 files changed, 33 insertions(+), 18 deletions(-) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 2988d582ffd..8589aa784f6 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -235,7 +235,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ // The snap state is exhausted, pass the entire key/val set for verification root := trieId.Root if origin == nil && !diskMore { - stackTr := trie.NewStackTrieWithOwner(nil, trieId.Owner) + stackTr := trie.NewStackTrie(nil) for i, key := range keys { stackTr.TryUpdate(key, vals[i]) } @@ -364,7 +364,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(trie.TrieID(common.Hash{}), snapTrieDb) + snapTrie := trie.NewEmpty(snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } diff --git a/trie/iterator.go b/trie/iterator.go index 724a7aaf11e..b13651fc043 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -376,7 +376,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } // Retrieve the specified node from the underlying node reader. - // it.trie.resolveHash is not used since in that function the + // it.trie.resolveAndTrack is not used since in that function the // loaded blob will be tracked, while it's not required here since // all loaded nodes won't be linked to trie at all and track nodes // may lead to out-of-memory issue. @@ -390,7 +390,7 @@ func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) } } // Retrieve the specified node from the underlying node reader. - // it.trie.resolveHash is not used since in that function the + // it.trie.resolveAndTrack is not used since in that function the // loaded blob will be tracked, while it's not required here since // all loaded nodes won't be linked to trie at all and track nodes // may lead to out-of-memory issue. diff --git a/trie/proof.go b/trie/proof.go index a95b60967ea..8e706f886b5 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -59,6 +59,11 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e key = key[1:] nodes = append(nodes, n) case hashNode: + // Retrieve the specified node from the underlying node reader. + // trie.resolveAndTrack is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. var err error tn, err = t.reader.node(prefix, common.BytesToHash(n)) if err != nil { diff --git a/trie/trie.go b/trie/trie.go index 64014578e5d..bec6a1cc789 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -108,7 +108,7 @@ func New(id *ID, db NodeReader) (*Trie, error) { //tracer: newTracer(), } if id.Root != (common.Hash{}) && id.Root != emptyRoot { - rootnode, err := trie.resolveHash(id.Root[:], nil) + rootnode, err := trie.resolveAndTrack(id.Root[:], nil) if err != nil { return nil, err } @@ -175,7 +175,7 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode } return value, n, didResolve, err case hashNode: - child, err := t.resolveHash(n, key[:pos]) + child, err := t.resolveAndTrack(n, key[:pos]) if err != nil { return nil, n, true, err } @@ -251,7 +251,7 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new return item, n, resolved, err case hashNode: - child, err := t.resolveHash(n, path[:pos]) + child, err := t.resolveAndTrack(n, path[:pos]) if err != nil { return nil, n, 1, err } @@ -372,7 +372,7 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error // We've hit a part of the trie that isn't loaded yet. Load // the node and insert into it. This leaves all child nodes on // the path to the value in the trie. - rn, err := t.resolveHash(n, prefix) + rn, err := t.resolveAndTrack(n, prefix) if err != nil { return false, nil, err } @@ -526,7 +526,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // We've hit a part of the trie that isn't loaded yet. Load // the node and delete from it. This leaves all child nodes on // the path to the value in the trie. - rn, err := t.resolveHash(n, prefix) + rn, err := t.resolveAndTrack(n, prefix) if err != nil { return false, nil, err } @@ -550,15 +550,16 @@ func concat(s1 []byte, s2 ...byte) []byte { func (t *Trie) resolve(n node, prefix []byte) (node, error) { if n, ok := n.(hashNode); ok { - return t.resolveHash(n, prefix) + return t.resolveAndTrack(n, prefix) } return n, nil } -// resolveHash loads node from the underlying store with the given node hash and -// path prefix. The rlp-encoded blob is preferred to be loaded from database -// because node pre-value is required to be tracked. -func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { +// resolveAndTrack loads node from the underlying store with the given node hash +// and path prefix and also tracks the loaded node blob in tracer treated as the +// node's original value. The rlp-encoded blob is preferred to be loaded from +// database because it's easy to decode node while complex to encode node to blob. +func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { blob, err := t.reader.nodeBlob(prefix, common.BytesToHash(n)) if err != nil { return nil, err diff --git a/trie/trie_test.go b/trie/trie_test.go index 1f3c8f3925f..063275929b3 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -439,10 +439,19 @@ func runRandTest(rt randTest) bool { rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } case opProve: - err := tr.Prove(step.key, 0, rawdb.NewMemoryDatabase()) + hash := tr.Hash() + if hash == emptyRoot { + continue + } + proofDb := rawdb.NewMemoryDatabase() + err := tr.Prove(step.key, 0, proofDb) if err != nil { rt[i].err = fmt.Errorf("failed for proving key %#x, %v", step.key, err) } + _, err = VerifyProof(hash, step.key, proofDb) + if err != nil { + rt[i].err = fmt.Errorf("failed for verifying key %#x, %v", step.key, err) + } case opHash: tr.Hash() case opCommit: @@ -483,7 +492,7 @@ func runRandTest(rt randTest) bool { // Enable node tracing. Resolve the root node again explicitly // since it's not captured at the beginning. tr.tracer = newTracer() - tr.resolveHash(root.Bytes(), nil) + tr.resolveAndTrack(root.Bytes(), nil) origTrie = tr.Copy() case opItercheckhash: diff --git a/trie/util_test.go b/trie/util_test.go index c9629899402..7086fbc4de0 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -160,7 +160,7 @@ func TestTrieTracePrevValue(t *testing.T) { } trie, _ = New(TrieID(root), db) trie.tracer = newTracer() - trie.resolveHash(root.Bytes(), nil) + trie.resolveAndTrack(root.Bytes(), nil) // Load all nodes in trie for _, val := range vals { @@ -222,7 +222,7 @@ func TestTrieTracePrevValue(t *testing.T) { // Delete entries from trie, ensure all previous values are correct. trie, _ = New(TrieID(root), db) trie.tracer = newTracer() - trie.resolveHash(root.Bytes(), nil) + trie.resolveAndTrack(root.Bytes(), nil) for _, val := range vals { trie.TryDelete([]byte(val.k)) From c720fcf2eaf094f82bdd565ca31578bb12d4dd2d Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 26 Sep 2022 15:33:48 +0800 Subject: [PATCH 07/10] trie: fix imports --- trie/util_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/util_test.go b/trie/util_test.go index 7086fbc4de0..e0e31420503 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -18,9 +18,9 @@ package trie import ( "bytes" - "github.com/ethereum/go-ethereum/common" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) From 11b4bc08a4dc4c431a9d00ade3010c63d5a9522e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Sep 2022 12:43:32 +0200 Subject: [PATCH 08/10] trie: add utility print-method for nodeset --- trie/nodeset.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/trie/nodeset.go b/trie/nodeset.go index 8e0a1fe5150..cb7a8de8811 100644 --- a/trie/nodeset.go +++ b/trie/nodeset.go @@ -150,6 +150,29 @@ func (set *NodeSet) Hashes() []common.Hash { return ret } +// Summary returns a string-representation of the NodeSet. +func (set *NodeSet) Summary() string { + var out = new(strings.Builder) + fmt.Fprintf(out, "nodeset owner: %v\n", set.owner) + if set.updates != nil { + for _, key := range set.updates.order { + updated := set.updates.nodes[key] + if updated.prev != nil { + fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", key, updated.hash, updated.prev) + } else { + fmt.Fprintf(out, " [+]: %x -> %v\n", key, updated.hash) + } + } + } + for k, n := range set.deletes { + fmt.Fprintf(out, " [-]: %x -> %x\n", k, n) + } + for _, n := range set.leaves { + fmt.Fprintf(out, "[leaf]: %v\n", n) + } + return out.String() +} + // MergedNodeSet represents a merged dirty node set for a group of tries. type MergedNodeSet struct { sets map[common.Hash]*NodeSet From 697bf71a128664470f3bdf784b3c036674b92b7d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Sep 2022 13:34:43 +0200 Subject: [PATCH 09/10] trie: import err --- trie/nodeset.go | 1 + 1 file changed, 1 insertion(+) diff --git a/trie/nodeset.go b/trie/nodeset.go index cb7a8de8811..ecf9a5ee670 100644 --- a/trie/nodeset.go +++ b/trie/nodeset.go @@ -19,6 +19,7 @@ package trie import ( "fmt" "reflect" + "strings" "github.com/ethereum/go-ethereum/common" ) From 983aa838a402aac3909a79e6b05975fbf6dc7619 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 26 Sep 2022 15:39:56 +0200 Subject: [PATCH 10/10] trie: fix go vet warnings --- trie/nodeset.go | 6 ++++++ trie/trie_test.go | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/trie/nodeset.go b/trie/nodeset.go index ecf9a5ee670..0f9d4ea0157 100644 --- a/trie/nodeset.go +++ b/trie/nodeset.go @@ -35,15 +35,18 @@ type memoryNode struct { // memoryNodeSize is the raw size of a memoryNode data structure without any // node data included. It's an approximate size, but should be a lot better // than not counting them. +// nolint:unused var memoryNodeSize = int(reflect.TypeOf(memoryNode{}).Size()) // memorySize returns the total memory size used by this node. +// nolint:unused func (n *memoryNode) memorySize(key int) int { return int(n.size) + memoryNodeSize + key } // rlp returns the raw rlp encoded blob of the cached trie node, either directly // from the cache, or by regenerating it from the collapsed node. +// nolint:unused func (n *memoryNode) rlp() []byte { if node, ok := n.node.(rawNode); ok { return node @@ -53,6 +56,7 @@ func (n *memoryNode) rlp() []byte { // obj returns the decoded and expanded trie node, either directly from the cache, // or by regenerating it from the rlp encoded blob. +// nolint:unused func (n *memoryNode) obj() node { if node, ok := n.node.(rawNode); ok { return mustDecodeNode(n.hash[:], node) @@ -67,12 +71,14 @@ type nodeWithPrev struct { } // unwrap returns the internal memoryNode object. +// nolint:unused func (n *nodeWithPrev) unwrap() *memoryNode { return n.memoryNode } // memorySize returns the total memory size used by this node. It overloads // the function in memoryNode by counting the size of previous value as well. +// nolint: unused func (n *nodeWithPrev) memorySize(key int) int { return n.memoryNode.memorySize(key) + len(n.prev) } diff --git a/trie/trie_test.go b/trie/trie_test.go index 063275929b3..d2a599ffdd6 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -468,7 +468,6 @@ func runRandTest(rt randTest) bool { if !bytes.Equal(blob, got) { rt[i].err = fmt.Errorf("prevalue mismatch for 0x%x, got 0x%x want 0x%x", path, got, blob) panic(rt[i].err) - return false } } for path, prev := range nodes.deletes {