simulators/portal/history/trin-bridge: add portal network history trin-bridge simulator

.gitignore

@@ -3,3 +3,7 @@ workspace
 .idea/
 # build output
 /hive
+
+# build output for rust simulators files
+simulators/**/Cargo.lock
+simulators/**/target

simulators/portal/history/trin-bridge/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "trin-bridge"
+version = "0.1.0"
+authors = ["Kolby ML (Moroz Liebl) <[email protected]>"]
+edition = "2021"
+
+[dependencies]
+ethportal-api = { git = "https://github.com/ethereum/trin", rev = "b530dc3d40d51c21c800089eacb2852ebd8c4d45" }
+portal-spec-test-utils-rs = { git = "https://github.com/ethereum/portal-spec-tests", rev = "d1e996d0d4dc2136b3cd38d9e25cdc3a6b74dcd9" }
+hivesim = { git = "https://github.com/ethereum/portal-hive", rev = "8ff1e3d3c941dd00d56dacd777a5dfb71edf402f" }
+itertools = "0.10.5"
+serde_yaml = "0.9"
+tokio = { version = "1", features = ["full"] }
+tracing = "0.1.37"
+tracing-subscriber = "0.3.16"

simulators/portal/history/trin-bridge/Dockerfile

@@ -0,0 +1,40 @@
+FROM rust:1.71.1 AS builder
+
+# create a new empty shell project
+RUN USER=root cargo new --bin trin-bridge
+WORKDIR /trin-bridge
+
+RUN apt-get update && apt-get install clang -y
+
+# copy over manifests and source to build image
+COPY Cargo.toml ./Cargo.toml
+COPY src ./src
+
+# build for release
+RUN cargo build --release
+
+# final base
+FROM ubuntu:22.04
+
+RUN apt update && apt install wget -y
+
+# Use these for amd-based devices
+RUN wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+RUN dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+# Use these for arm-based devices
+#RUN wget http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb
+#RUN dpkg -i libssl1.1_1.1.1f-1ubuntu2_arm64.deb
+
+# copy build artifacts from build stage
+COPY --from=builder /trin-bridge/target/release/trin-bridge .
+ADD https://raw.githubusercontent.com/ethereum/portal-spec-tests/master/tests/mainnet/history/hive/test_data_collection_of_forks_blocks.yaml ./test-data/test_data_collection_of_forks_blocks.yaml
+
+RUN apt-get update && apt-get install libcurl4 -y
+
+ENV RUST_LOG=debug
+
+# Fake secrets for Trin bridge activation, not actually used in the used `BridgeMode::Test` mode
+ENV PANDAOPS_CLIENT_ID=xxx
+ENV PANDAOPS_CLIENT_SECRET=xxx
+
+ENTRYPOINT ["./trin-bridge"]

simulators/portal/history/trin-bridge/src/constants.rs

@@ -0,0 +1,4 @@
+pub const TRIN_BRIDGE_CLIENT_TYPE: &str = "trin-bridge";
+pub const BOOTNODES_ENVIRONMENT_VARIABLE: &str = "HIVE_BOOTNODES";
+pub const HIVE_CHECK_LIVE_PORT: &str = "HIVE_CHECK_LIVE_PORT";
+pub const TEST_DATA_FILE_PATH: &str = "./test-data/test_data_collection_of_forks_blocks.yaml";

simulators/portal/history/trin-bridge/src/main.rs

@@ -0,0 +1,183 @@
+mod constants;
+
+use crate::constants::{
+    BOOTNODES_ENVIRONMENT_VARIABLE, HIVE_CHECK_LIVE_PORT, TEST_DATA_FILE_PATH,
+    TRIN_BRIDGE_CLIENT_TYPE,
+};
+use ethportal_api::HistoryContentKey;
+use ethportal_api::HistoryContentValue;
+use ethportal_api::PossibleHistoryContentValue;
+use ethportal_api::{Discv5ApiClient, HistoryNetworkApiClient};
+use hivesim::types::ClientDefinition;
+use hivesim::{dyn_async, Client, NClientTestSpec, Simulation, Suite, Test, TestSpec};
+use itertools::Itertools;
+use portal_spec_test_utils_rs::get_flair;
+use serde_yaml::Value;
+use std::collections::HashMap;
+use tokio::time::Duration;
+
+fn process_content(content: Vec<(HistoryContentKey, HistoryContentValue)>) -> Vec<String> {
+    let mut last_header = content.first().unwrap().clone();
+
+    let mut result: Vec<String> = vec![];
+    for history_content in content.into_iter() {
+        if let HistoryContentKey::BlockHeaderWithProof(_) = &history_content.0 {
+            last_header = history_content.clone();
+        }
+        let comment =
+            if let HistoryContentValue::BlockHeaderWithProof(header_with_proof) = &last_header.1 {
+                let content_type = match &history_content.0 {
+                    HistoryContentKey::BlockHeaderWithProof(_) => "header".to_string(),
+                    HistoryContentKey::BlockBody(_) => "body".to_string(),
+                    HistoryContentKey::BlockReceipts(_) => "receipt".to_string(),
+                    HistoryContentKey::EpochAccumulator(_) => "epoch accumulator".to_string(),
+                };
+                format!(
+                    "{}{} {}",
+                    header_with_proof.header.number,
+                    get_flair(header_with_proof.header.number),
+                    content_type
+                )
+            } else {
+                unreachable!("History test dated is formatted incorrectly")
+            };
+        result.push(comment)
+    }
+    result
+}
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt::init();
+
+    let mut suite = Suite {
+        name: "trin-bridge-tests".to_string(),
+        description: "The portal bridge test suite".to_string(),
+        tests: vec![],
+    };
+
+    suite.add(TestSpec {
+        name: "Trin bridge tests".to_string(),
+        description: "".to_string(),
+        always_run: false,
+        run: test_portal_bridge,
+        client: None,
+    });
+
+    let sim = Simulation::new();
+    run_suite(sim, suite).await;
+}
+
+async fn run_suite(host: Simulation, suite: Suite) {
+    let name = suite.clone().name;
+    let description = suite.clone().description;
+
+    let suite_id = host.start_suite(name, description, "".to_string()).await;
+
+    for test in &suite.tests {
+        test.run_test(host.clone(), suite_id, suite.clone()).await;
+    }
+
+    host.end_suite(suite_id).await;
+}
+
+dyn_async! {
+   async fn test_portal_bridge<'a> (test: &'a mut Test, _client: Option<Client>) {
+        // Get all available portal clients
+        let clients = test.sim.client_types().await;
+        if !clients.iter().any(|client_definition| client_definition.name == *TRIN_BRIDGE_CLIENT_TYPE) {
+            panic!("This simulator is required to be ran with client `trin-bridge`")
+        }
+        let clients: Vec<ClientDefinition> = clients.into_iter().filter(|client| client.name != *TRIN_BRIDGE_CLIENT_TYPE).collect();
+
+        // Iterate over all possible pairings of clients and run the tests (including self-pairings)
+        for (client_a, client_b) in clients.iter().cartesian_product(clients.iter()) {
+            test.run(
+                NClientTestSpec {
+                    name: format!("Bridge test. A:{} --> B:{}", client_a.name, client_b.name),
+                    description: "".to_string(),
+                    always_run: false,
+                    run: test_bridge,
+                    environments: None,
+                    test_data: None,
+                    clients: vec![client_a.clone(), client_b.clone()],
+                }
+            ).await;
+        }
+   }
+}
+
+dyn_async! {
+    async fn test_bridge<'a>(clients: Vec<Client>, _: Option<Vec<(String, String)>>) {
+        let (client_a, client_b) = match clients.iter().collect_tuple() {
+            Some((client_a, client_b)) => (client_a, client_b),
+            None => {
+                panic!("Unable to get expected amount of clients from NClientTestSpec");
+            }
+        };
+
+        let client_b_enr = match client_b.rpc.node_info().await {
+            Ok(node_info) => node_info.enr,
+            Err(err) => {
+                panic!("Error getting node info: {err:?}");
+            }
+        };
+        match HistoryNetworkApiClient::add_enr(&client_a.rpc, client_b_enr.clone()).await {
+            Ok(response) => if !response {
+                panic!("AddEnr expected to get true and instead got false")
+            },
+            Err(err) => panic!("{}", &err.to_string()),
+        }
+
+        let client_a_enr = match client_a.rpc.node_info().await {
+            Ok(node_info) => node_info.enr,
+            Err(err) => {
+                panic!("Error getting node info: {err:?}");
+            }
+        };
+        client_a.test.start_client(TRIN_BRIDGE_CLIENT_TYPE.to_string(), Some(HashMap::from([(BOOTNODES_ENVIRONMENT_VARIABLE.to_string(), client_a_enr.to_base64()), (HIVE_CHECK_LIVE_PORT.to_string(), 0.to_string())]))).await;
+
+
+
+        // With default node settings nodes should be storing all content
+        let values = std::fs::read_to_string(TEST_DATA_FILE_PATH)
+            .expect("cannot find test asset");
+        let values: Value = serde_yaml::from_str(&values).unwrap();
+        let content_vec: Vec<(HistoryContentKey, HistoryContentValue)> = values.as_sequence().unwrap().iter().map(|value| {
+            let content_key: HistoryContentKey =
+                serde_yaml::from_value(value.get("content_key").unwrap().clone()).unwrap();
+            let content_value: HistoryContentValue =
+                serde_yaml::from_value(value.get("content_value").unwrap().clone()).unwrap();
+            (content_key, content_value)
+        }).collect();
+        let comments = process_content(content_vec.clone());
+
+        // wait content_vec.len() seconds for data to propagate, giving more time if more items are propagating
+        tokio::time::sleep(Duration::from_secs(content_vec.len() as u64)).await;
+
+        let mut result = vec![];
+        for (index, (content_key, content_value)) in content_vec.into_iter().enumerate() {
+            match client_b.rpc.local_content(content_key.clone()).await {
+                Ok(possible_content) => {
+                   match possible_content {
+                        PossibleHistoryContentValue::ContentPresent(content) => {
+                            if content != content_value {
+                                result.push(format!("Error content received for block {} was different then expected", comments[index]));
+                            }
+                        }
+                        PossibleHistoryContentValue::ContentAbsent => {
+                            result.push(format!("Error content for block {} was absent", comments[index]));
+                        }
+                    }
+                }
+                Err(err) => {
+                    panic!("Unable to get received content: {err:?}");
+                }
+            }
+        }
+
+        if !result.is_empty() {
+            panic!("Client B: {:?}", result);
+        }
+    }
+}