diff --git a/README.md b/README.md index f202adfa..356c22d9 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,13 @@ KVVM is served over RPC with [go-plugin](https://github.com/hashicorp/go-plugin) # Features TODO: Extend on * PoW Transactions (no tokens) +* No Nonces (replay protection from blockID + txID) * Prefixes (address prefixes reserved) * Hashed Value Keys * Prefix Expiry (based on weight of all key-values) * Load Units vs Fee Units * Lifeline Rewards (why run a node -> don't need to mine) +* Block Value Reuse # RPC ## /public @@ -72,6 +74,21 @@ curl --location --request POST 'http://localhost:61858/ext/bc/BJfusM2TpHCEfmt5i7 {"jsonrpc":"2.0","result":{"success":true},"id":1} COMMENT +# resolve a path +curl --location --request POST 'http://localhost:61858/ext/bc/BJfusM2TpHCEfmt5i7qeE1MwVCbw5jU1TcZNz8MYUwG1PGYRL/public' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "jsonrpc": "2.0", + "method": "quarkvm.resolve", + "params":{ + "path": "patrick.avax/twitter" + }, + "id": 1 +}' +< 0 { + if err := db.Put(PrefixTxValueKey(id), tp.Value); err != nil { + t.Fatal(err) + } + } + } + err := tv.utx.Execute(db, uint64(tv.blockTime), id) if !errors.Is(err, tv.err) { t.Fatalf("#%d: tx.Execute err expected %v, got %v", i, tv.err, err) } diff --git a/chain/storage.go b/chain/storage.go index 68284b6f..44674d5c 100644 --- a/chain/storage.go +++ b/chain/storage.go @@ -9,6 +9,7 @@ import ( "errors" "math" + "github.com/ava-labs/avalanchego/cache" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/hashing" @@ -20,28 +21,38 @@ import ( // TODO: cleanup mapping diagram // 0x0/ (block hashes) // 0x1/ (tx hashes) -// 0x2/ (singleton prefix info) +// -> [tx hash]=>nil +// 0x2/ (tx values) +// -> [tx hash]=>value +// 0x3/ (singleton prefix info) // -> [prefix]:[prefix info/raw prefix] -// 0x3/ (prefix keys) +// 0x4/ (prefix keys) // -> [raw prefix] // -> [key] -// 0x4/ (prefix expiry queue) +// 0x5/ (prefix expiry queue) // -> [raw prefix] -// 0x5/ (prefix pruning queue) +// 0x6/ (prefix pruning queue) // -> [raw prefix] const ( blockPrefix = 0x0 txPrefix = 0x1 - infoPrefix = 0x2 - keyPrefix = 0x3 - expiryPrefix = 0x4 - pruningPrefix = 0x5 + txValuePrefix = 0x2 + infoPrefix = 0x3 + keyPrefix = 0x4 + expiryPrefix = 0x5 + pruningPrefix = 0x6 shortIDLen = 20 + + linkedTxLRUSize = 512 ) -var lastAccepted = []byte("last_accepted") +var ( + lastAccepted = []byte("last_accepted") + + linkedTxCache = &cache.LRU{Size: linkedTxLRUSize} +) // [blockPrefix] + [delimiter] + [blockID] func PrefixBlockKey(blockID ids.ID) (k []byte) { @@ -61,6 +72,15 @@ func PrefixTxKey(txID ids.ID) (k []byte) { return k } +// [txValuePrefix] + [delimiter] + [txID] +func PrefixTxValueKey(txID ids.ID) (k []byte) { + k = make([]byte, 2+len(txID)) + k[0] = txValuePrefix + k[1] = parser.Delimiter + copy(k[2:], txID[:]) + return k +} + // [infoPrefix] + [delimiter] + [prefix] func PrefixInfoKey(prefix []byte) (k []byte) { k = make([]byte, 2+len(prefix)) @@ -167,19 +187,95 @@ func GetValue(db database.KeyValueReader, prefix []byte, key []byte) ([]byte, bo // [keyPrefix] + [delimiter] + [rawPrefix] + [delimiter] + [key] k := PrefixValueKey(prefixInfo.RawPrefix, key) - v, err := db.Get(k) + txID, err := db.Get(k) if errors.Is(err, database.ErrNotFound) { return nil, false, nil } + if err != nil { + return nil, false, err + } + + // Lookup stored value + v, err := getLinkedValue(db, txID) + if err != nil { + return nil, false, err + } return v, true, err } +// linkValues extracts all *SetTx.Value in [block] and replaces them with the +// corresponding txID where they were found. The extracted value is then +// written to disk. +func linkValues(db database.KeyValueWriter, block *StatelessBlock) ([]*Transaction, error) { + ogTxs := make([]*Transaction, len(block.Txs)) + for i, tx := range block.Txs { + switch t := tx.UnsignedTransaction.(type) { + case *SetTx: + if len(t.Value) == 0 { + ogTxs[i] = tx + continue + } + + // Copy transaction for later + cptx := tx.Copy() + if err := cptx.Init(); err != nil { + return nil, err + } + ogTxs[i] = cptx + + if err := db.Put(PrefixTxValueKey(tx.ID()), t.Value); err != nil { + return nil, err + } + t.Value = tx.id[:] // used to properly parse on restore + default: + ogTxs[i] = tx + } + } + return ogTxs, nil +} + +// restoreValues restores the unlinked values associated with all *SetTx.Value +// in [block]. +func restoreValues(db database.KeyValueReader, block *StatefulBlock) error { + for _, tx := range block.Txs { + if t, ok := tx.UnsignedTransaction.(*SetTx); ok { + if len(t.Value) == 0 { + continue + } + txID, err := ids.ToID(t.Value) + if err != nil { + return err + } + b, err := db.Get(PrefixTxValueKey(txID)) + if err != nil { + return err + } + t.Value = b + } + } + return nil +} + func SetLastAccepted(db database.KeyValueWriter, block *StatelessBlock) error { bid := block.ID() if err := db.Put(lastAccepted, bid[:]); err != nil { return err } - return db.Put(PrefixBlockKey(bid), block.Bytes()) + ogTxs, err := linkValues(db, block) + if err != nil { + return err + } + sbytes, err := Marshal(block.StatefulBlock) + if err != nil { + return err + } + if err := db.Put(PrefixBlockKey(bid), sbytes); err != nil { + return err + } + // Restore the original transactions in the block in case it is cached for + // later use. + block.Txs = ogTxs + return nil } func HasLastAccepted(db database.Database) (bool, error) { @@ -197,8 +293,23 @@ func GetLastAccepted(db database.KeyValueReader) (ids.ID, error) { return ids.ToID(v) } -func GetBlock(db database.KeyValueReader, bid ids.ID) ([]byte, error) { - return db.Get(PrefixBlockKey(bid)) +func GetBlock(db database.KeyValueReader, bid ids.ID) (*StatefulBlock, []byte, error) { + b, err := db.Get(PrefixBlockKey(bid)) + if err != nil { + return nil, nil, err + } + blk := new(StatefulBlock) + if _, err := Unmarshal(b, blk); err != nil { + return nil, nil, err + } + if err := restoreValues(db, blk); err != nil { + return nil, nil, err + } + fb, err := Marshal(blk) + if err != nil { + return nil, nil, err + } + return blk, fb, nil } // ExpireNext queries "expiryPrefix" key space to find expiring keys, @@ -361,11 +472,7 @@ func DeletePrefixKey(db database.Database, prefix []byte, key []byte) error { func SetTransaction(db database.KeyValueWriter, tx *Transaction) error { k := PrefixTxKey(tx.ID()) - b, err := Marshal(tx) - if err != nil { - return err - } - return db.Put(k, b) + return db.Put(k, nil) } func HasTransaction(db database.KeyValueReader, txID ids.ID) (bool, error) { @@ -378,6 +485,24 @@ type KeyValue struct { Value []byte `serialize:"true" json:"value"` } +func getLinkedValue(db database.KeyValueReader, b []byte) ([]byte, error) { + bh := string(b) + if v, ok := linkedTxCache.Get(bh); ok { + return v.([]byte), nil + } + txID, err := ids.ToID(b) + if err != nil { + return nil, err + } + vk := PrefixTxValueKey(txID) + v, err := db.Get(vk) + if err != nil { + return nil, err + } + linkedTxCache.Put(bh, v) + return v, nil +} + // Range reads keys from the store. func Range(db database.Database, prefix []byte, key []byte, opts ...OpOption) (kvs []KeyValue, err error) { prefixInfo, exists, err := GetPrefixInfo(db, prefix) @@ -414,9 +539,13 @@ func Range(db database.Database, prefix []byte, key []byte, opts ...OpOption) (k comp := bytes.Compare(startKey, curKey) if comp == 0 { // startKey == curKey + v, err := getLinkedValue(db, cursor.Value()) + if err != nil { + return nil, err + } kvs = append(kvs, KeyValue{ Key: formattedKey, - Value: cursor.Value(), + Value: v, }) continue } @@ -432,9 +561,13 @@ func Range(db database.Database, prefix []byte, key []byte, opts ...OpOption) (k break } + v, err := getLinkedValue(db, cursor.Value()) + if err != nil { + return nil, err + } kvs = append(kvs, KeyValue{ Key: formattedKey, - Value: cursor.Value(), + Value: v, }) } return kvs, nil diff --git a/chain/storage_test.go b/chain/storage_test.go index 38588b10..0db4a303 100644 --- a/chain/storage_test.go +++ b/chain/storage_test.go @@ -157,11 +157,15 @@ func TestRange(t *testing.T) { } for i := 0; i < 5; i++ { + id := ids.GenerateTestID() + if err := db.Put(PrefixTxValueKey(id), []byte(fmt.Sprintf("bar%05d", i))); err != nil { + t.Fatal(err) + } if err := PutPrefixKey( db, pfx, []byte(fmt.Sprintf("hello%05d", i)), - []byte(fmt.Sprintf("bar%05d", i)), + id[:], ); err != nil { t.Fatal(err) } diff --git a/chain/tx.go b/chain/tx.go index 9f0b2729..98ed5eb3 100644 --- a/chain/tx.go +++ b/chain/tx.go @@ -29,6 +29,15 @@ func NewTx(utx UnsignedTransaction, sig []byte) *Transaction { } } +func (t *Transaction) Copy() *Transaction { + sig := make([]byte, len(t.Signature)) + copy(sig, t.Signature) + return &Transaction{ + UnsignedTransaction: t.UnsignedTransaction.Copy(), + Signature: sig, + } +} + func UnsignedBytes(utx UnsignedTransaction) ([]byte, error) { b, err := Marshal(utx) if err != nil { @@ -109,7 +118,7 @@ func (t *Transaction) Execute(db database.Database, blockTime int64, context *Co if !pk.Verify(t.unsignedBytes, t.Signature) { return ErrInvalidSignature } - if err := t.UnsignedTransaction.Execute(db, uint64(blockTime)); err != nil { + if err := t.UnsignedTransaction.Execute(db, uint64(blockTime), t.id); err != nil { return err } return SetTransaction(db, t) diff --git a/chain/unsigned_tx.go b/chain/unsigned_tx.go index 1597f3fd..357af83f 100644 --- a/chain/unsigned_tx.go +++ b/chain/unsigned_tx.go @@ -19,5 +19,5 @@ type UnsignedTransaction interface { LoadUnits() uint64 // units that should impact fee rate ExecuteBase() error - Execute(database.Database, uint64) error + Execute(database.Database, uint64, ids.ID) error } diff --git a/client/client.go b/client/client.go index 40009fb1..5e486a83 100644 --- a/client/client.go +++ b/client/client.go @@ -24,11 +24,11 @@ type Client interface { Ping() (bool, error) // Returns the corresponding prefix information. PrefixInfo(pfx []byte) (*chain.PrefixInfo, error) - // Preferred fetches the ID of the currently preferred block. - Preferred() (ids.ID, error) - // Checks the validity of the block. + // Accepted fetches the ID of the last accepted block. + Accepted() (ids.ID, error) + // Checks the validity of the blockID. // Returns "true" if the block is valid. - CheckBlock(blkID ids.ID) (bool, error) + ValidBlockID(blkID ids.ID) (bool, error) // Requests for the estimated difficulty from VM. EstimateDifficulty() (uint64, uint64, error) // Issues the transaction and returns the transaction ID. @@ -39,10 +39,10 @@ type Client interface { PollTx(ctx context.Context, txID ids.ID) (confirmed bool, err error) // Range runs range-query and returns the results. Range(pfx, key []byte, opts ...OpOption) (kvs []chain.KeyValue, err error) + // Resolve returns the value associated with a path + Resolve(path string) (exists bool, value []byte, err error) // Performs Proof-of-Work (PoW) by enumerating the graffiti. - Mine( - ctx context.Context, utx chain.UnsignedTransaction, - ) (chain.UnsignedTransaction, error) + Mine(ctx context.Context, utx chain.UnsignedTransaction) (chain.UnsignedTransaction, error) } // New creates a new client object. @@ -85,11 +85,11 @@ func (cli *client) PrefixInfo(pfx []byte) (*chain.PrefixInfo, error) { return resp.Info, nil } -func (cli *client) Preferred() (ids.ID, error) { - resp := new(vm.CurrBlockReply) +func (cli *client) Accepted() (ids.ID, error) { + resp := new(vm.LastAcceptedReply) if err := cli.req.SendRequest( - "currBlock", - &vm.CurrBlockArgs{}, + "lastAccepted", + nil, resp, ); err != nil { color.Red("failed to get curr block %v", err) @@ -98,7 +98,7 @@ func (cli *client) Preferred() (ids.ID, error) { return resp.BlockID, nil } -func (cli *client) CheckBlock(blkID ids.ID) (bool, error) { +func (cli *client) ValidBlockID(blkID ids.ID) (bool, error) { resp := new(vm.ValidBlockIDReply) if err := cli.req.SendRequest( "validBlockID", @@ -193,6 +193,20 @@ done: return false, ctx.Err() } +func (cli *client) Resolve(path string) (exists bool, value []byte, err error) { + resp := new(vm.ResolveReply) + if err = cli.req.SendRequest( + "resolve", + &vm.ResolveArgs{ + Path: path, + }, + resp, + ); err != nil { + return false, nil, err + } + return resp.Exists, resp.Value, nil +} + type Op struct { key []byte rangeEnd []byte diff --git a/client/miner.go b/client/miner.go index e31c1143..20e3e802 100644 --- a/client/miner.go +++ b/client/miner.go @@ -148,7 +148,7 @@ func (cli *client) Mine(ctx context.Context, utx chain.UnsignedTransaction) (cha for { select { case <-t.C: - blkID, err := cli.Preferred() + blkID, err := cli.Accepted() if err != nil { return err } diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 610944e4..3cd482a7 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -125,10 +125,10 @@ var _ = ginkgo.Describe("[Ping]", func() { }) var _ = ginkgo.Describe("[Claim/SetTx]", func() { - ginkgo.It("get currently preferred block ID", func() { + ginkgo.It("get currently accepted block ID", func() { for _, inst := range instances { cli := inst.cli - _, err := cli.Preferred() + _, err := cli.Accepted() gomega.Ω(err).Should(gomega.BeNil()) } }) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 100a6755..89a1800b 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -187,10 +187,10 @@ var _ = ginkgo.Describe("[Ping]", func() { }) var _ = ginkgo.Describe("[ClaimTx]", func() { - ginkgo.It("get currently preferred block ID", func() { + ginkgo.It("get currently accepted block ID", func() { for _, inst := range instances { cli := inst.cli - _, err := cli.Preferred() + _, err := cli.Accepted() gomega.Ω(err).Should(gomega.BeNil()) } }) @@ -328,6 +328,13 @@ var _ = ginkgo.Describe("[ClaimTx]", func() { gomega.Ω(kvs[0].Key).To(gomega.Equal(k)) gomega.Ω(kvs[0].Value).To(gomega.Equal(v)) }) + + ginkgo.By("read back from VM with resolve", func() { + exists, value, err := instances[0].cli.Resolve(string(pfx) + "/" + string(k)) + gomega.Ω(err).To(gomega.BeNil()) + gomega.Ω(exists).To(gomega.BeTrue()) + gomega.Ω(value).To(gomega.Equal(v)) + }) }) ginkgo.It("fail Gossip ClaimTx to a stale node when missing previous blocks", func() { diff --git a/vm/public_service.go b/vm/public_service.go index ce691417..2c22d9c3 100644 --- a/vm/public_service.go +++ b/vm/public_service.go @@ -12,6 +12,7 @@ import ( log "github.com/inconshreveable/log15" "github.com/ava-labs/quarkvm/chain" + "github.com/ava-labs/quarkvm/parser" ) var ( @@ -85,14 +86,12 @@ func (svc *PublicService) CheckTx(_ *http.Request, args *CheckTxArgs, reply *Che return nil } -type CurrBlockArgs struct{} - -type CurrBlockReply struct { +type LastAcceptedReply struct { BlockID ids.ID `serialize:"true" json:"blockId"` } -func (svc *PublicService) CurrBlock(_ *http.Request, args *CurrBlockArgs, reply *CurrBlockReply) error { - reply.BlockID = svc.vm.preferred +func (svc *PublicService) LastAccepted(_ *http.Request, _ *struct{}, reply *LastAcceptedReply) error { + reply.BlockID = svc.vm.lastAccepted.ID() return nil } @@ -191,3 +190,28 @@ func (svc *PublicService) Range(_ *http.Request, args *RangeArgs, reply *RangeRe reply.KeyValues = kvs return nil } + +type ResolveArgs struct { + Path string `serialize:"true" json:"path"` +} + +type ResolveReply struct { + Exists bool `serialize:"true" json:"exists"` + Value []byte `serialize:"true" json:"value"` +} + +func (svc *PublicService) Resolve(_ *http.Request, args *ResolveArgs, reply *ResolveReply) error { + pfx, key, _, err := parser.ParsePrefixKey( + []byte(args.Path), + parser.WithCheckPrefix(), + parser.WithCheckKey(), + ) + if err != nil { + return err + } + + v, exists, err := chain.GetValue(svc.vm.db, pfx, key) + reply.Exists = exists + reply.Value = v + return err +} diff --git a/vm/vm.go b/vm/vm.go index 76246908..03c443ae 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -104,7 +104,7 @@ type VM struct { } const ( - blocksLRUSize = 100 + blocksLRUSize = 128 ) // implements "snowmanblock.ChainVM.common.VM" @@ -331,12 +331,12 @@ func (vm *VM) GetStatelessBlock(blkID ids.ID) (*chain.StatelessBlock, error) { } // not found in memory, fetch from disk if accepted - bytes, err := chain.GetBlock(vm.db, blkID) + stBlk, bytes, err := chain.GetBlock(vm.db, blkID) if err != nil { return nil, err } // If block on disk, it must've been accepted - return chain.ParseBlock(bytes, choices.Accepted, vm) + return chain.ParseStatefulBlock(stBlk, bytes, choices.Accepted, vm) } // implements "snowmanblock.ChainVM.commom.VM.Parser"