Increase GetState::<ActorState> default timeout and make it configurable (#1474)

Summary:
Pull Request resolved: #1474

The current timeout of 1 second for `GetState::<ActorState>` for supervision was causing at least one test to fail. This diff makes the value configurable, and also increases the default to 30 seconds. The failing test now passes.

Example test failure that shows the supervision timeout: P1984316048
ghstack-source-id: 315174069

Reviewed By: mariusae

Differential Revision: D84232284

fbshipit-source-id: 4a56b45e4d0d828602883a68dd57bc044639b4dd

Author: samlurye

hyperactor_mesh/src/v1/actor_mesh.rs

@@ -405,6 +405,7 @@ mod tests {
     use crate::v1::ActorMeshRef;
     use crate::v1::Name;
     use crate::v1::ProcMesh;
+    use crate::v1::proc_mesh::GET_ACTOR_STATE_MAX_IDLE;
     use crate::v1::testactor;
     use crate::v1::testing;
 
@@ -577,6 +578,9 @@ mod tests {
     async fn test_actor_states_with_process_exit() {
         hyperactor_telemetry::initialize_logging_for_test();
 
+        let config = hyperactor::config::global::lock();
+        let _guard = config.override_key(GET_ACTOR_STATE_MAX_IDLE, Duration::from_secs(1));
+
         let instance = testing::instance().await;
         // Listen for supervision events sent to the parent instance.
         let (supervision_port, mut supervision_receiver) =

hyperactor_mesh/src/v1/proc_mesh.rs

@@ -72,6 +72,9 @@ declare_attrs! {
     /// The maximum idle time between updates while spawning actor meshes.
     @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_ACTOR_SPAWN_MAX_IDLE".to_string())
     pub attr ACTOR_SPAWN_MAX_IDLE: Duration = Duration::from_secs(30);
+
+    @meta(CONFIG_ENV_VAR = "HYPERACTOR_MESH_GET_ACTOR_STATE_MAX_IDLE".to_string())
+    pub attr GET_ACTOR_STATE_MAX_IDLE: Duration = Duration::from_secs(30);
 }
 
 /// A reference to a single [`hyperactor::Proc`].
@@ -545,7 +548,9 @@ impl ProcMeshRef {
             // the agent will be unresponsive.
             // We handle this by setting a timeout on the recv, and if we don't get a
             // message we assume the agent is dead and return a failed state.
-            let state = RealClock.timeout(Duration::from_secs(1), rx.recv()).await;
+            let state = RealClock
+                .timeout(config::global::get(GET_ACTOR_STATE_MAX_IDLE), rx.recv())
+                .await;
             if let Ok(state) = state {
                 // Handle non-timeout receiver error.
                 let state = state?;

python/tests/test_actor_error.py

@@ -778,7 +778,7 @@ async def test_supervision_with_proc_mesh_stopped(mesh, api_ver: ApiVersion) ->
 # TODO - re-enable after resolving T232206970
 @pytest.mark.oss_skip
 @pytest.mark.parametrize("api_ver", [ApiVersion.V0, ApiVersion.V1])
-@pytest.mark.timeout(30)
+@pytest.mark.timeout(60)
 async def test_supervision_with_sending_error(api_ver: ApiVersion) -> None:
     # Messages of length > this will cause a send error and a returned
     # undeliverable.