Skip to content

Commit 60e30a9

Browse files
rjl493456442rjl493456442
authored
core/rawdb: refactor db inspector for extending multiple ancient store (#25896)
This PR ports a few changes from PBSS: - Fix the snapshot generator waiter in case the generation is not even initialized - Refactor db inspector for ancient store
1 parent a1fc0d8 commit 60e30a9

File tree

9 files changed

+247
-73
lines changed

9 files changed

+247
-73
lines changed

‎core/blockchain.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ const (
123123
BlockChainVersion uint64 = 8
124124
)
125125

126-
// CacheConfig contains the configuration values for the trie caching/pruning
126+
// CacheConfig contains the configuration values for the trie database
127127
// that's resident in a blockchain.
128128
type CacheConfig struct {
129129
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
@@ -1408,7 +1408,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
14081408
if len(logs) > 0 {
14091409
bc.logsFeed.Send(logs)
14101410
}
1411-
// In theory we should fire a ChainHeadEvent when we inject
1411+
// In theory, we should fire a ChainHeadEvent when we inject
14121412
// a canonical block, but sometimes we can insert a batch of
14131413
// canonical blocks. Avoid firing too many ChainHeadEvents,
14141414
// we will fire an accumulated ChainHeadEvent and disable fire

‎core/blockchain_test.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4007,3 +4007,89 @@ func TestTxIndexer(t *testing.T) {
40074007
os.RemoveAll(frdir)
40084008
}
40094009
}
4010+
4011+
func TestCreateThenDeletePreByzantium(t *testing.T) {
4012+
// We use Ropsten chain config instead of Testchain config, this is
4013+
// deliberate: we want to use pre-byz rules where we have intermediate state roots
4014+
// between transactions.
4015+
testCreateThenDelete(t, params.RopstenChainConfig)
4016+
}
4017+
func TestCreateThenDeletePostByzantium(t *testing.T) {
4018+
testCreateThenDelete(t, params.TestChainConfig)
4019+
}
4020+
4021+
// testCreateThenDelete tests a creation and subsequent deletion of a contract, happening
4022+
// within the same block.
4023+
func testCreateThenDelete(t *testing.T, config *params.ChainConfig) {
4024+
var (
4025+
engine = ethash.NewFaker()
4026+
// A sender who makes transactions, has some funds
4027+
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
4028+
address = crypto.PubkeyToAddress(key.PublicKey)
4029+
destAddress = crypto.CreateAddress(address, 0)
4030+
funds = big.NewInt(1000000000000000)
4031+
)
4032+
4033+
// runtime code is 0x60ffff : PUSH1 0xFF SELFDESTRUCT, a.k.a SELFDESTRUCT(0xFF)
4034+
code := append([]byte{0x60, 0xff, 0xff}, make([]byte, 32-3)...)
4035+
initCode := []byte{
4036+
// SSTORE 1:1
4037+
byte(vm.PUSH1), 0x1,
4038+
byte(vm.PUSH1), 0x1,
4039+
byte(vm.SSTORE),
4040+
// Get the runtime-code on the stack
4041+
byte(vm.PUSH32)}
4042+
initCode = append(initCode, code...)
4043+
initCode = append(initCode, []byte{
4044+
byte(vm.PUSH1), 0x0, // offset
4045+
byte(vm.MSTORE),
4046+
byte(vm.PUSH1), 0x3, // size
4047+
byte(vm.PUSH1), 0x0, // offset
4048+
byte(vm.RETURN), // return 3 bytes of zero-code
4049+
}...)
4050+
gspec := &Genesis{
4051+
Config: config,
4052+
Alloc: GenesisAlloc{
4053+
address: {Balance: funds},
4054+
},
4055+
}
4056+
nonce := uint64(0)
4057+
signer := types.HomesteadSigner{}
4058+
_, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) {
4059+
fee := big.NewInt(1)
4060+
if b.header.BaseFee != nil {
4061+
fee = b.header.BaseFee
4062+
}
4063+
b.SetCoinbase(common.Address{1})
4064+
tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{
4065+
Nonce: nonce,
4066+
GasPrice: new(big.Int).Set(fee),
4067+
Gas: 100000,
4068+
Data: initCode,
4069+
})
4070+
nonce++
4071+
b.AddTx(tx)
4072+
tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{
4073+
Nonce: nonce,
4074+
GasPrice: new(big.Int).Set(fee),
4075+
Gas: 100000,
4076+
To: &destAddress,
4077+
})
4078+
b.AddTx(tx)
4079+
nonce++
4080+
})
4081+
// Import the canonical chain
4082+
chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{
4083+
//Debug: true,
4084+
//Tracer: logger.NewJSONLogger(nil, os.Stdout),
4085+
}, nil, nil)
4086+
if err != nil {
4087+
t.Fatalf("failed to create tester chain: %v", err)
4088+
}
4089+
// Import the blocks
4090+
for _, block := range blocks {
4091+
if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
4092+
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
4093+
}
4094+
}
4095+
}

‎core/rawdb/ancient_scheme.go‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616

1717
package rawdb
1818

19-
import "fmt"
20-
2119
// The list of table names of chain freezer.
2220
const (
2321
// chainFreezerHeaderTable indicates the name of the freezer header table.
@@ -53,34 +51,3 @@ var (
5351

5452
// freezers the collections of all builtin freezers.
5553
var freezers = []string{chainFreezerName}
56-
57-
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
58-
// ancient indicates the path of root ancient directory where the chain freezer can
59-
// be opened. Start and end specify the range for dumping out indexes.
60-
// Note this function can only be used for debugging purposes.
61-
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
62-
var (
63-
path string
64-
tables map[string]bool
65-
)
66-
switch freezerName {
67-
case chainFreezerName:
68-
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
69-
default:
70-
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
71-
}
72-
noSnappy, exist := tables[tableName]
73-
if !exist {
74-
var names []string
75-
for name := range tables {
76-
names = append(names, name)
77-
}
78-
return fmt.Errorf("unknown table, supported ones: %v", names)
79-
}
80-
table, err := newFreezerTable(path, tableName, noSnappy, true)
81-
if err != nil {
82-
return err
83-
}
84-
table.dumpIndexStdout(start, end)
85-
return nil
86-
}

‎core/rawdb/ancient_utils.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// Copyright 2022 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package rawdb
18+
19+
import (
20+
"fmt"
21+
22+
"github.com/ethereum/go-ethereum/common"
23+
"github.com/ethereum/go-ethereum/ethdb"
24+
)
25+
26+
type tableSize struct {
27+
name string
28+
size common.StorageSize
29+
}
30+
31+
// freezerInfo contains the basic information of the freezer.
32+
type freezerInfo struct {
33+
name string // The identifier of freezer
34+
head uint64 // The number of last stored item in the freezer
35+
tail uint64 // The number of first stored item in the freezer
36+
sizes []tableSize // The storage size per table
37+
}
38+
39+
// count returns the number of stored items in the freezer.
40+
func (info *freezerInfo) count() uint64 {
41+
return info.head - info.tail + 1
42+
}
43+
44+
// size returns the storage size of the entire freezer.
45+
func (info *freezerInfo) size() common.StorageSize {
46+
var total common.StorageSize
47+
for _, table := range info.sizes {
48+
total += table.size
49+
}
50+
return total
51+
}
52+
53+
// inspectFreezers inspects all freezers registered in the system.
54+
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
55+
var infos []freezerInfo
56+
for _, freezer := range freezers {
57+
switch freezer {
58+
case chainFreezerName:
59+
// Chain ancient store is a bit special. It's always opened along
60+
// with the key-value store, inspect the chain store directly.
61+
info := freezerInfo{name: freezer}
62+
// Retrieve storage size of every contained table.
63+
for table := range chainFreezerNoSnappy {
64+
size, err := db.AncientSize(table)
65+
if err != nil {
66+
return nil, err
67+
}
68+
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
69+
}
70+
// Retrieve the number of last stored item
71+
ancients, err := db.Ancients()
72+
if err != nil {
73+
return nil, err
74+
}
75+
info.head = ancients - 1
76+
77+
// Retrieve the number of first stored item
78+
tail, err := db.Tail()
79+
if err != nil {
80+
return nil, err
81+
}
82+
info.tail = tail
83+
infos = append(infos, info)
84+
85+
default:
86+
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
87+
}
88+
}
89+
return infos, nil
90+
}
91+
92+
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
93+
// ancient indicates the path of root ancient directory where the chain freezer can
94+
// be opened. Start and end specify the range for dumping out indexes.
95+
// Note this function can only be used for debugging purposes.
96+
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
97+
var (
98+
path string
99+
tables map[string]bool
100+
)
101+
switch freezerName {
102+
case chainFreezerName:
103+
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
104+
default:
105+
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
106+
}
107+
noSnappy, exist := tables[tableName]
108+
if !exist {
109+
var names []string
110+
for name := range tables {
111+
names = append(names, name)
112+
}
113+
return fmt.Errorf("unknown table, supported ones: %v", names)
114+
}
115+
table, err := newFreezerTable(path, tableName, noSnappy, true)
116+
if err != nil {
117+
return err
118+
}
119+
table.dumpIndexStdout(start, end)
120+
return nil
121+
}

‎core/rawdb/database.go‎

Lines changed: 18 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"os"
2424
"path"
25+
"strings"
2526
"sync/atomic"
2627
"time"
2728

@@ -379,13 +380,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
379380
beaconHeaders stat
380381
cliqueSnaps stat
381382

382-
// Ancient store statistics
383-
ancientHeadersSize common.StorageSize
384-
ancientBodiesSize common.StorageSize
385-
ancientReceiptsSize common.StorageSize
386-
ancientTdsSize common.StorageSize
387-
ancientHashesSize common.StorageSize
388-
389383
// Les statistic
390384
chtTrieNodes stat
391385
bloomTrieNodes stat
@@ -473,20 +467,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
473467
logged = time.Now()
474468
}
475469
}
476-
// Inspect append-only file store then.
477-
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
478-
for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
479-
if size, err := db.AncientSize(category); err == nil {
480-
*ancientSizes[i] += common.StorageSize(size)
481-
total += common.StorageSize(size)
482-
}
483-
}
484-
// Get number of ancient rows inside the freezer
485-
ancients := counter(0)
486-
if count, err := db.Ancients(); err == nil {
487-
ancients = counter(count)
488-
}
489-
// Display the database statistic.
470+
// Display the database statistic of key-value store.
490471
stats := [][]string{
491472
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
492473
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
@@ -504,14 +485,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
504485
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
505486
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
506487
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
507-
{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
508-
{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
509-
{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
510-
{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
511-
{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
512488
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
513489
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
514490
}
491+
// Inspect all registered append-only file store then.
492+
ancients, err := inspectFreezers(db)
493+
if err != nil {
494+
return err
495+
}
496+
for _, ancient := range ancients {
497+
for _, table := range ancient.sizes {
498+
stats = append(stats, []string{
499+
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
500+
strings.Title(table.name),
501+
table.size.String(),
502+
fmt.Sprintf("%d", ancient.count()),
503+
})
504+
}
505+
total += ancient.size()
506+
}
515507
table := tablewriter.NewWriter(os.Stdout)
516508
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
517509
table.SetFooter([]string{"", "Total", total.String(), " "})
@@ -521,6 +513,5 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
521513
if unaccounted.size > 0 {
522514
log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
523515
}
524-
525516
return nil
526517
}

‎core/state/snapshot/snapshot.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,9 @@ type Tree struct {
187187
// If the memory layers in the journal do not match the disk layer (e.g. there is
188188
// a gap) or the journal is missing, there are two repair cases:
189189
//
190-
// - if the 'recovery' parameter is true, all memory diff-layers will be discarded.
191-
// This case happens when the snapshot is 'ahead' of the state trie.
190+
// - if the 'recovery' parameter is true, memory diff-layers and the disk-layer
191+
// will all be kept. This case happens when the snapshot is 'ahead' of the
192+
// state trie.
192193
// - otherwise, the entire snapshot is considered invalid and will be recreated on
193194
// a background thread.
194195
func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) {
@@ -199,16 +200,16 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root
199200
triedb: triedb,
200201
layers: make(map[common.Hash]snapshot),
201202
}
202-
// Create the building waiter iff the background generation is allowed
203-
if !config.NoBuild && !config.AsyncBuild {
204-
defer snap.waitBuild()
205-
}
206203
// Attempt to load a previously persisted snapshot and rebuild one if failed
207204
head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild)
208205
if disabled {
209206
log.Warn("Snapshot maintenance disabled (syncing)")
210207
return snap, nil
211208
}
209+
// Create the building waiter iff the background generation is allowed
210+
if !config.NoBuild && !config.AsyncBuild {
211+
defer snap.waitBuild()
212+
}
212213
if err != nil {
213214
log.Warn("Failed to load snapshot", "err", err)
214215
if !config.NoBuild {

0 commit comments

Comments
 (0)