From 61d978c73c5e527f50be1b57a8a11f0f6fab19ef Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Apr 2022 09:15:58 +0200 Subject: [PATCH 1/2] cmd/geth: inspect snapshot dangling storage --- cmd/geth/snapshot.go | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index d0539eeff74..6ed925a0a30 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "os" "time" @@ -102,6 +103,25 @@ geth snapshot verify-state will traverse the whole accounts and storages set based on the specified snapshot and recalculate the root hash of state for verification. In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "check-dangling-storage", + Usage: "Check that there is no 'dangling' snap storage", + ArgsUsage: "", + Action: utils.MigrateFlags(checkDanglingStorage), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.SepoliaFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +geth snapshot check-dangling-storage traverses the snap storage +data, and verifies that all snapshot storage data has a corresponding account. `, }, { @@ -245,6 +265,69 @@ func verifyState(ctx *cli.Context) error { return nil } +// checkDanglingStorage iterates the snap storage data, and verifies that all +// storage also has corresponding account data. +func checkDanglingStorage(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chaindb := utils.MakeChainDatabase(ctx, stack, true) + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + var root = headBlock.Root() + if ctx.NArg() == 1 { + root, err = parseRoot(ctx.Args()[0]) + if err != nil { + log.Error("Failed to resolve state root", "err", err) + return err + } + } + var ( + lastReport = time.Now() + start = time.Now() + snapshot = snaptree.Snapshot(root) + lastKey []byte + it = rawdb.NewKeyLengthIterator(chaindb.NewIterator(rawdb.SnapshotStoragePrefix, nil), 1+2*common.HashLength) + ) + defer it.Release() + for it.Next() { + k := it.Key() + accKey := k[1:33] + if bytes.Equal(accKey, lastKey) { + // No need to look up for every slot + continue + } + lastKey = common.CopyBytes(accKey) + if time.Since(lastReport) > time.Second*8 { + log.Info("Iterating snap storage", "at", fmt.Sprintf("%#x", accKey), "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + data, err := snapshot.AccountRLP(common.BytesToHash(accKey)) + if err != nil { + log.Error("Error loading snap storage data", "account", fmt.Sprintf("%#x", accKey), "err", err) + continue + } + if len(data) == 0 { + log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accKey), "storagekey", fmt.Sprintf("%#x", k)) + continue + } + } + log.Info("Verified the snapshot storage", "root", root, "time", common.PrettyDuration(time.Since(start)), "err", it.Error()) + return nil +} + // traverseState is a helper function used for pruning verification. // Basically it just iterates the trie, ensure all nodes and associated // contract codes are present. From 9f2626aa4b3c23e9708d500702a8c78aed2e04de Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Apr 2022 11:08:04 +0200 Subject: [PATCH 2/2] cmd/geth: make verify-state invoke verify-dangling --- cmd/geth/snapshot.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 6ed925a0a30..9be50a20c57 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -262,6 +263,10 @@ func verifyState(ctx *cli.Context) error { return err } log.Info("Verified the state", "root", root) + if err := checkDangling(chaindb, snaptree.Snapshot(root)); err != nil { + log.Error("Dangling snap storage check failed", "root", root, "err", err) + return err + } return nil } @@ -294,10 +299,14 @@ func checkDanglingStorage(ctx *cli.Context) error { return err } } + return checkDangling(chaindb, snaptree.Snapshot(root)) +} + +func checkDangling(chaindb ethdb.Database, snap snapshot.Snapshot) error { + log.Info("Checking dangling snapshot storage") var ( lastReport = time.Now() start = time.Now() - snapshot = snaptree.Snapshot(root) lastKey []byte it = rawdb.NewKeyLengthIterator(chaindb.NewIterator(rawdb.SnapshotStoragePrefix, nil), 1+2*common.HashLength) ) @@ -314,17 +323,17 @@ func checkDanglingStorage(ctx *cli.Context) error { log.Info("Iterating snap storage", "at", fmt.Sprintf("%#x", accKey), "elapsed", common.PrettyDuration(time.Since(start))) lastReport = time.Now() } - data, err := snapshot.AccountRLP(common.BytesToHash(accKey)) + data, err := snap.AccountRLP(common.BytesToHash(accKey)) if err != nil { log.Error("Error loading snap storage data", "account", fmt.Sprintf("%#x", accKey), "err", err) - continue + return err } if len(data) == 0 { log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accKey), "storagekey", fmt.Sprintf("%#x", k)) - continue + return fmt.Errorf("dangling snapshot storage account %#x", accKey) } } - log.Info("Verified the snapshot storage", "root", root, "time", common.PrettyDuration(time.Since(start)), "err", it.Error()) + log.Info("Verified the snapshot storage", "root", snap.Root(), "time", common.PrettyDuration(time.Since(start)), "err", it.Error()) return nil }