Fix FleetManager behavior when create-fleet returns no instances

In case of ICE create-fleet doesn't raise an exception and the default behavior isn't triggered
This change forces the failure reason to ICE when create-fleet doens't return any instance

Signed-off-by: Nicola Sirena <[email protected]>

Author: NSsirena

CHANGELOG.md

@@ -6,8 +6,8 @@ This file is used to list changes made in each version of the aws-parallelcluste
 3.6.1
 ------
 
-**CHANGES**
-- There were no changes for this version.
+**BUG FIXES**
+- Fix fast insufficient capacity fail-over logic when using Multiple Instance Types and no instances are returned
 
 3.6.0
 ------

src/slurm_plugin/instance_manager.py

@@ -167,11 +167,20 @@ def _update_slurm_node_addrs(self, slurm_nodes, launched_instances):
                     print_with_count(zip(launched_nodes, launched_instances)),
                 )
             if fail_launch_nodes:
-                logger.warning(
-                    "Failed to launch instances due to limited EC2 capacity for following nodes: %s",
-                    print_with_count(fail_launch_nodes),
-                )
-                self._update_failed_nodes(set(fail_launch_nodes), "LimitedInstanceCapacity")
+                if launched_nodes:
+                    logger.warning(
+                        "Failed to launch instances due to limited EC2 capacity for following nodes: %s",
+                        print_with_count(fail_launch_nodes),
+                    )
+                    self._update_failed_nodes(set(fail_launch_nodes), "LimitedInstanceCapacity")
+                else:
+                    # EC2 Fleet doens't trigger any exception in case of ICEs and may return more than one error
+                    # for each request. So when no instances were launched we force the reason to ICE
+                    logger.error(
+                        "Failed to launch instances due to limited EC2 capacity for following nodes: %s",
+                        print_with_count(fail_launch_nodes),
+                    )
+                    self._update_failed_nodes(set(fail_launch_nodes), "InsufficientInstanceCapacity")
 
             return dict(zip(launched_nodes, launched_instances))
 

tests/slurm_plugin/test_fleet_manager.py

@@ -219,6 +219,32 @@ def test_launch_instances(self, boto3_stubber, launch_params, mocked_boto3_reque
 
 # -------- Ec2CreateFleetManager ------
 
+test_fleet_exception_params = {
+    "LaunchTemplateConfigs": [
+        {
+            "LaunchTemplateSpecification": {"LaunchTemplateName": "hit-queue1-fleet-spot", "Version": "$Latest"},
+            "Overrides": [
+                {
+                    "InstanceRequirements": {
+                        "VCpuCount": {"Min": 2},
+                        "MemoryMiB": {"Min": 2048},
+                        "AllowedInstanceTypes": ["inf*"],
+                        "AcceleratorManufacturers": ["nvidia"],
+                    }
+                }
+            ],
+        }
+    ],
+    "SpotOptions": {
+        "AllocationStrategy": "capacity-optimized",
+        "SingleInstanceType": False,
+        "SingleAvailabilityZone": True,
+        "MinTargetCapacity": 1,
+    },
+    "TargetCapacitySpecification": {"TotalTargetCapacity": 5, "DefaultTargetCapacityType": "spot"},
+    "Type": "instant",
+}
+
 test_fleet_spot_params = {
     "LaunchTemplateConfigs": [
         {
@@ -486,6 +512,26 @@ def test_evaluate_launch_params(
                     },
                 ],
             ),
+            # create-fleet - exception
+            (
+                test_fleet_exception_params,
+                [
+                    MockedBoto3Request(
+                        method="create_fleet",
+                        response={
+                            "Instances": [],
+                            "Errors": [
+                                {"ErrorCode": "InvalidParameterValue", "ErrorMessage": "Insufficient capacity."}
+                            ],
+                            "ResponseMetadata": {"RequestId": "1234-abcde"},
+                        },
+                        expected_params=test_fleet_exception_params,
+                        generate_error=True,
+                        error_code="InvalidParameterValue",
+                    ),
+                ],
+                [],
+            ),
             # normal - on-demand
             (
                 test_on_demand_params,
@@ -549,7 +595,7 @@ def test_evaluate_launch_params(
                 ],
             ),
         ],
-        ids=["fleet_spot", "fleet_ondemand"],
+        ids=["fleet_spot", "fleet_exception", "fleet_ondemand"],
     )
     def test_launch_instances(
         self,
@@ -567,8 +613,13 @@ def test_launch_instances(
             "hit", "region", "boto3_config", FLEET_CONFIG, "queue2", "fleet-ondemand", False, {}, {}
         )
 
-        assigned_nodes = fleet_manager._launch_instances(launch_params)
-        assert_that(assigned_nodes.get("Instances", [])).is_equal_to(expected_assigned_nodes)
+        if mocked_boto3_request[0].generate_error:
+            with pytest.raises(Exception) as e:
+                fleet_manager._launch_instances(launch_params)
+                assert isinstance(e, ClientError)
+        else:
+            assigned_nodes = fleet_manager._launch_instances(launch_params)
+            assert_that(assigned_nodes.get("Instances", [])).is_equal_to(expected_assigned_nodes)
 
     @pytest.mark.parametrize(
         ("instance_ids", "mocked_boto3_request", "expected_exception", "expected_error", "expected_result"),

tests/slurm_plugin/test_instance_manager.py

@@ -814,7 +814,7 @@ def test_add_instances(
                 ["queue1-st-c5xlarge-1"],
                 {},
                 None,
-                {"LimitedInstanceCapacity": {"queue1-st-c5xlarge-1"}},
+                {"InsufficientInstanceCapacity": {"queue1-st-c5xlarge-1"}},
                 False,
                 "dns.domain",
             ),