From d84f9cf0473777dcd9a70134dbbddda4e65f4afd Mon Sep 17 00:00:00 2001 From: Riley Dulin Date: Fri, 3 Oct 2025 14:48:33 -0700 Subject: [PATCH] Turn on cargo test for the hyperactor crate (#1417) Summary: Pull Request resolved: https://github.com/meta-pytorch/monarch/pull/1417 We should have Rust tests run in Github CI to ensure we don't break them. Before we were blocked on issues with `cargo test` running tests in threads, and we need process isolation because our tests modify things like environment variables. Use `cargo nextest` which is a package that runs test in parallel processes instead of threads. For now, just run these in a GPU environment. A cpu-only run may be beneficial for either cost savings, faster iteration cycles, or for test coverage itself. But I'll add that later. Some tests are failing in Github but not internally. Add a feature to skip these called "fb". Reviewed By: mariusae Differential Revision: D83856634 --- .github/workflows/test-gpu-rust.yml | 4 ++++ hyperactor/src/channel.rs | 4 ++++ hyperactor/src/channel/net.rs | 18 ++++++++++++++++++ hyperactor/src/config.rs | 2 ++ hyperactor/src/host.rs | 2 ++ scripts/common-setup.sh | 4 ++++ 6 files changed, 34 insertions(+) diff --git a/.github/workflows/test-gpu-rust.yml b/.github/workflows/test-gpu-rust.yml index ea9dbc054..8c8c24441 100644 --- a/.github/workflows/test-gpu-rust.yml +++ b/.github/workflows/test-gpu-rust.yml @@ -58,3 +58,7 @@ jobs: echo "Running OSS Rust tests..." # TODO: fix broken tests, then update to `cargo test --no-fail-fast` cargo test -p monarch_rdma + # Uses cargo nextest to run tests in separate processes, which better matches + # internal buck test behavior. + # TODO: increase coverage to more crates. + cargo nextest run -p hyperactor --no-fail-fast diff --git a/hyperactor/src/channel.rs b/hyperactor/src/channel.rs index 8904ee82a..8e3558f77 100644 --- a/hyperactor/src/channel.rs +++ b/hyperactor/src/channel.rs @@ -919,6 +919,8 @@ mod tests { } #[tokio::test] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_dial_serve() { for addr in addrs() { let (listen_addr, mut rx) = crate::channel::serve::(addr).unwrap(); @@ -929,6 +931,8 @@ mod tests { } #[tokio::test] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_send() { let config = crate::config::global::lock(); diff --git a/hyperactor/src/channel/net.rs b/hyperactor/src/channel/net.rs index 010a5d233..a656c6b6d 100644 --- a/hyperactor/src/channel/net.rs +++ b/hyperactor/src/channel/net.rs @@ -2559,6 +2559,8 @@ mod tests { #[tracing_test::traced_test] #[async_timed_test(timeout_secs = 30)] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_tcp_basic() { let (addr, mut rx) = tcp::serve::("[::1]:0".parse().unwrap()).unwrap(); { @@ -2581,6 +2583,8 @@ mod tests { // The message size is limited by CODEC_MAX_FRAME_LENGTH. #[async_timed_test(timeout_secs = 5)] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_tcp_message_size() { let default_size_in_bytes = 100 * 1024 * 1024; // Use temporary config for this test @@ -2610,6 +2614,8 @@ mod tests { } #[async_timed_test(timeout_secs = 30)] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_ack_flush() { let config = config::global::lock(); // Set a large value to effectively prevent acks from being sent except @@ -2632,6 +2638,8 @@ mod tests { #[tracing_test::traced_test] #[tokio::test] + // TODO: OSS: failed to retrieve ipv6 address + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_meta_tls_basic() { let addr = ChannelAddr::any(ChannelTransport::MetaTls(TlsMode::IpV6)); let meta_addr = match addr { @@ -3235,6 +3243,8 @@ mod tests { #[tracing_test::traced_test] #[tokio::test] + // TODO: OSS: The logs_assert function returned an error: expected log not found + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_tcp_tx_delivery_timeout() { // This link always fails to connect. let link = MockLink::::fail_connects(); @@ -3659,12 +3669,16 @@ mod tests { #[tracing_test::traced_test] #[async_timed_test(timeout_secs = 30)] + // TODO: OSS: The logs_assert function returned an error: expected log not found + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_ack_exceeded_limit_with_connected_link() { verify_ack_exceeded_limit(false).await; } #[tracing_test::traced_test] #[async_timed_test(timeout_secs = 30)] + // TODO: OSS: The logs_assert function returned an error: expected log not found + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_ack_exceeded_limit_with_broken_link() { verify_ack_exceeded_limit(true).await; } @@ -3834,6 +3848,8 @@ mod tests { } #[async_timed_test(timeout_secs = 300)] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }) + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_tcp_throughput() { let config = config::global::lock(); let _guard = @@ -3884,6 +3900,8 @@ mod tests { #[tracing_test::traced_test] #[async_timed_test(timeout_secs = 60)] + // TODO: OSS: The logs_assert function returned an error: expected log not found + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_net_tx_closed_on_server_reject() { let link = MockLink::::new(); let receiver_storage = link.receiver_storage(); diff --git a/hyperactor/src/config.rs b/hyperactor/src/config.rs index ff1f9c1aa..b44ab7d8d 100644 --- a/hyperactor/src/config.rs +++ b/hyperactor/src/config.rs @@ -369,6 +369,8 @@ mod tests { #[tracing_test::traced_test] #[test] + // TODO: OSS: The logs_assert function returned an error: missing log lines: {"# export HYPERACTOR_DEFAULT_ENCODING=serde_multipart", ...} + #[cfg_attr(not(feature = "fb"), ignore)] fn test_from_env() { // Set environment variables // SAFETY: TODO: Audit that the environment access only happens in single-threaded code. diff --git a/hyperactor/src/host.rs b/hyperactor/src/host.rs index dd861c0b7..ec6d4600a 100644 --- a/hyperactor/src/host.rs +++ b/hyperactor/src/host.rs @@ -1158,6 +1158,8 @@ mod tests { } #[tokio::test] + // TODO: OSS: called `Result::unwrap()` on an `Err` value: ReadFailed { manifest_path: "/meta-pytorch/monarch/target/debug/deps/hyperactor-0e1fe83af739d976.resources.json", source: Os { code: 2, kind: NotFound, message: "No such file or directory" } } + #[cfg_attr(not(feature = "fb"), ignore)] async fn test_process_proc_manager() { hyperactor_telemetry::initialize_logging(crate::clock::ClockKind::default()); diff --git a/scripts/common-setup.sh b/scripts/common-setup.sh index 69152744f..8c265582d 100644 --- a/scripts/common-setup.sh +++ b/scripts/common-setup.sh @@ -33,6 +33,10 @@ setup_rust_toolchain() { source "${HOME}"/.cargo/env rustup toolchain install nightly rustup default nightly + # We use cargo nextest to run tests in individual processes for similarity + # to buck test. + # Replace "cargo test" commands with "cargo nextest run". + cargo install cargo-nextest --locked } install_build_dependencies() {