Replace old headers with CHT in light clients (#512)

* storage proofs

* CHT

Author: svyatonik

Cargo.lock

@@ -8,7 +8,6 @@ error-chain = "0.12"
 fnv = "1.0"
 log = "0.3"
 parking_lot = "0.4"
-triehash = "0.1"
 hex-literal = "0.1"
 futures = "0.1.17"
 ed25519 = { path = "../ed25519" }
@@ -27,6 +26,7 @@ substrate-telemetry = { path = "../telemetry" }
 hashdb = { git = "https://github.com/paritytech/parity-common" }
 patricia-trie = { git = "https://github.com/paritytech/parity-common" }
 rlp = { git = "https://github.com/paritytech/parity-common" }
+triehash = { git = "https://github.com/paritytech/parity-common" }
 
 [dev-dependencies]
 substrate-test-client = { path = "../test-client" }

substrate/client/Cargo.toml

@@ -61,7 +61,8 @@ use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
 use executor::RuntimeInfo;
 use state_machine::{CodeExecutor, DBValue, ExecutionStrategy};
-use utils::{Meta, db_err, meta_keys, number_to_db_key, open_database, read_db, read_id, read_meta};
+use utils::{Meta, db_err, meta_keys, number_to_db_key, db_key_to_number, open_database,
+	read_db, read_id, read_meta};
 use state_db::StateDb;
 pub use state_db::PruningMode;
 
@@ -183,6 +184,14 @@ impl<Block: BlockT> client::blockchain::HeaderBackend<Block> for BlockchainDb<Bl
 		}
 	}
 
+	fn number(&self, hash: Block::Hash) -> Result<Option<<Block::Header as HeaderT>::Number>, client::error::Error> {
+		read_id::<Block>(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash))
+			.and_then(|key| match key {
+				Some(key) => Ok(Some(db_key_to_number(&key)?)),
+				None => Ok(None),
+			})
+	}
+
 	fn hash(&self, number: <Block::Header as HeaderT>::Number) -> Result<Option<Block::Hash>, client::error::Error> {
 		read_db::<Block>(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x|
 			x.map(|raw| HashFor::<Block>::hash(&raw[..])).map(Into::into)
@@ -397,13 +406,13 @@ impl<Block> client::backend::Backend<Block, KeccakHasher, RlpCodec> for Backend<
 					}
 				};
 				if let Some(finalizing_hash) = finalizing_hash {
-					trace!("Finalizing block #{} ({:?})", number_u64 - self.finalization_window, finalizing_hash);
+					trace!(target: "db", "Finalizing block #{} ({:?})", number_u64 - self.finalization_window, finalizing_hash);
 					let commit = self.storage.state_db.finalize_block(&finalizing_hash);
 					apply_state_commit(&mut transaction, commit);
 				}
 			}
 
-			debug!("DB Commit {:?} ({}), best = {}", hash, number, pending_block.is_best);
+			debug!(target: "db", "DB Commit {:?} ({}), best = {}", hash, number, pending_block.is_best);
 			self.storage.db.write(transaction).map_err(db_err)?;
 			self.blockchain.update_meta(hash, number, pending_block.is_best);
 		}

substrate/client/db/src/lib.rs

@@ -23,25 +23,29 @@ use kvdb::{KeyValueDB, DBTransaction};
 
 use client::blockchain::{BlockStatus, Cache as BlockchainCache,
 	HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo};
+use client::cht;
 use client::error::{ErrorKind as ClientErrorKind, Result as ClientResult};
 use client::light::blockchain::Storage as LightBlockchainStorage;
 use codec::{Decode, Encode};
-use primitives::AuthorityId;
+use primitives::{AuthorityId, H256, KeccakHasher};
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor, Zero, As};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor,
+	Zero, One, As, NumberFor};
 use cache::DbCache;
-use utils::{meta_keys, Meta, db_err, number_to_db_key, open_database, read_db, read_id, read_meta};
+use utils::{meta_keys, Meta, db_err, number_to_db_key, db_key_to_number, open_database,
+	read_db, read_id, read_meta};
 use DatabaseSettings;
 
 pub(crate) mod columns {
 	pub const META: Option<u32> = ::utils::COLUMN_META;
 	pub const BLOCK_INDEX: Option<u32> = Some(1);
 	pub const HEADER: Option<u32> = Some(2);
 	pub const AUTHORITIES: Option<u32> = Some(3);
+	pub const CHT: Option<u32> = Some(4);
 }
 
 /// Keep authorities for last 'AUTHORITIES_ENTRIES_TO_KEEP' blocks.
-pub(crate) const AUTHORITIES_ENTRIES_TO_KEEP: u64 = 2048;
+pub(crate) const AUTHORITIES_ENTRIES_TO_KEEP: u64 = cht::SIZE;
 
 /// Light blockchain storage. Stores most recent headers + CHTs for older headers.
 pub struct LightStorage<Block: BlockT> {
@@ -146,6 +150,14 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
 		}
 	}
 
+	fn number(&self, hash: Block::Hash) -> ClientResult<Option<<<Block as BlockT>::Header as HeaderT>::Number>> {
+		read_id::<Block>(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash))
+			.and_then(|key| match key {
+				Some(key) => Ok(Some(db_key_to_number(&key)?)),
+				None => Ok(None),
+			})
+	}
+
 	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Option<Block::Hash>> {
 		read_db::<Block>(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x|
 			x.map(|raw| HashFor::<Block>::hash(&raw[..])).map(Into::into)
@@ -156,6 +168,7 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
 impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 	where
 		Block: BlockT,
+		Block::Hash: From<H256>,
 {
 	fn import_header(&self, is_new_best: bool, header: Block::Header, authorities: Option<Vec<AuthorityId>>) -> ClientResult<()> {
 		let mut transaction = DBTransaction::new();
@@ -187,6 +200,27 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 			None
 		};
 
+		// build new CHT if required
+		if let Some(new_cht_number) = cht::is_build_required(cht::SIZE, *header.number()) {
+			let new_cht_start: NumberFor<Block> = cht::start_number(cht::SIZE, new_cht_number);
+			let new_cht_root: Option<Block::Hash> = cht::compute_root::<Block::Header, KeccakHasher, _>(
+				cht::SIZE, new_cht_number, (new_cht_start.as_()..)
+				.map(|num| self.hash(As::sa(num)).unwrap_or_default()));
+
+			if let Some(new_cht_root) = new_cht_root {
+				transaction.put(columns::CHT, &number_to_db_key(new_cht_start), new_cht_root.as_ref());
+
+				let mut prune_block = new_cht_start;
+				let new_cht_end = cht::end_number(cht::SIZE, new_cht_number);
+				trace!(target: "db", "Replacing blocks [{}..{}] with CHT#{}", new_cht_start, new_cht_end, new_cht_number);
+
+				while prune_block <= new_cht_end {
+					transaction.delete(columns::HEADER, &number_to_db_key(prune_block));
+ 					prune_block += <<Block as BlockT>::Header as HeaderT>::Number::one();
+				}
+			}
+		}
+
 		debug!("Light DB Commit {:?} ({})", hash, number);
 		self.db.write(transaction).map_err(db_err)?;
 		self.update_meta(hash, number, is_new_best);
@@ -197,13 +231,24 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 		Ok(())
 	}
 
+	fn cht_root(&self, cht_size: u64, block: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Block::Hash> {
+		let no_cht_for_block = || ClientErrorKind::Backend(format!("CHT for block {} not exists", block)).into();
+
+		let cht_number = cht::block_to_cht_number(cht_size, block).ok_or_else(no_cht_for_block)?;
+		let cht_start = cht::start_number(cht_size, cht_number);
+		self.db.get(columns::CHT, &number_to_db_key(cht_start)).map_err(db_err)?
+			.ok_or_else(no_cht_for_block)
+			.and_then(|hash| Block::Hash::decode(&mut &*hash).ok_or_else(no_cht_for_block))
+	}
+
 	fn cache(&self) -> Option<&BlockchainCache<Block>> {
 		Some(&self.cache)
 	}
 }
 
 #[cfg(test)]
 pub(crate) mod tests {
+	use client::cht;
 	use runtime_primitives::testing::{H256 as Hash, Header, Block as RawBlock};
 	use super::*;
 
@@ -289,4 +334,59 @@ pub(crate) mod tests {
 		assert_eq!(db.db.iter(columns::HEADER).count(), 2);
 		assert_eq!(db.db.iter(columns::BLOCK_INDEX).count(), 2);
 	}
+
+	#[test]
+	fn ancient_headers_are_replaced_with_cht() {
+		let db = LightStorage::new_test();
+
+		// insert genesis block header (never pruned)
+		let mut prev_hash = insert_block(&db, &Default::default(), 0, None);
+
+		// insert SIZE blocks && ensure that nothing is pruned
+		for number in 0..cht::SIZE {
+			prev_hash = insert_block(&db, &prev_hash, 1 + number, None);
+		}
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 0);
+
+		// insert next SIZE blocks && ensure that nothing is pruned
+		for number in 0..cht::SIZE {
+			prev_hash = insert_block(&db, &prev_hash, 1 + cht::SIZE + number, None);
+		}
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + cht::SIZE) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 0);
+
+		// insert block #{2 * cht::SIZE + 1} && check that new CHT is created + headers of this CHT are pruned
+		insert_block(&db, &prev_hash, 1 + cht::SIZE + cht::SIZE, None);
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + 1) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 1);
+		assert!((0..cht::SIZE).all(|i| db.db.get(columns::HEADER, &number_to_db_key(1 + i)).unwrap().is_none()));
+	}
+
+	#[test]
+	fn get_cht_fails_for_genesis_block() {
+		assert!(LightStorage::<Block>::new_test().cht_root(cht::SIZE, 0).is_err());
+	}
+
+	#[test]
+	fn get_cht_fails_for_non_existant_cht() {
+		assert!(LightStorage::<Block>::new_test().cht_root(cht::SIZE, (cht::SIZE / 2) as u64).is_err());
+	}
+
+	#[test]
+	fn get_cht_works() {
+		let db = LightStorage::new_test();
+
+		// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
+		let mut prev_hash = Default::default();
+		for i in 0..1 + cht::SIZE + cht::SIZE + 1 {
+			prev_hash = insert_block(&db, &prev_hash, i as u64, None);
+		}
+
+		let cht_root_1 = db.cht_root(cht::SIZE, cht::start_number(cht::SIZE, 0)).unwrap();
+		let cht_root_2 = db.cht_root(cht::SIZE, (cht::start_number(cht::SIZE, 0) + cht::SIZE / 2) as u64).unwrap();
+		let cht_root_3 = db.cht_root(cht::SIZE, cht::end_number(cht::SIZE, 0)).unwrap();
+		assert_eq!(cht_root_1, cht_root_2);
+		assert_eq!(cht_root_2, cht_root_3);
+	}
 }

substrate/client/db/src/light.rs

@@ -31,6 +31,8 @@ pub trait HeaderBackend<Block: BlockT>: Send + Sync {
 	fn info(&self) -> Result<Info<Block>>;
 	/// Get block status.
 	fn status(&self, id: BlockId<Block>) -> Result<BlockStatus>;
+	/// Get block number by hash. Returns `None` if the header is not in the chain.
+	fn number(&self, hash: Block::Hash) -> Result<Option<<<Block as BlockT>::Header as HeaderT>::Number>>;
 	/// Get block hash by number. Returns `None` if the header is not in the chain.
 	fn hash(&self, number: NumberFor<Block>) -> Result<Option<Block::Hash>>;