Skips some tests that are currently causing problems for internal users (#266)

Co-authored-by: Auto-format Bot <[email protected]>

Author: brandon-wada

Makefile

@@ -37,11 +37,16 @@ TEST_ARGS=
 CLOUD_FILTERS = -m "not run_only_for_edge_endpoint"
 EDGE_FILTERS = -m "not skip_for_edge_endpoint"
 
+# Record information about the slowest 25 tests (but don't show anything slower than 0.1 seconds)
+PROFILING_ARGS = \
+	--durations 25 \
+	--durations-min 0.1
+
 test: install  ## Run tests against the prod API (needs GROUNDLIGHT_API_TOKEN)
-	${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test
+	${PYTEST} ${PROFILING_ARGS} ${TEST_ARGS} ${CLOUD_FILTERS} test
 
 test-4edge: install  ## Run tests against the prod API via the edge-endpoint (needs GROUNDLIGHT_API_TOKEN)
-	${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test
+	${PYTEST} ${PROFILING_ARGS} ${TEST_ARGS} ${EDGE_FILTERS} test
 
 test-local: install  ## Run tests against a localhost API (needs GROUNDLIGHT_API_TOKEN and a local API server)
 	GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test

test/unit/test_actions.py

@@ -13,9 +13,6 @@ def test_create_action(gl_experimental: ExperimentalApi):
     rule = gl_experimental.create_rule(det, f"test_rule_{name}", "EMAIL", "[email protected]")
     rule2 = gl_experimental.get_rule(rule.id)
     assert rule == rule2
-    gl_experimental.delete_rule(rule.id)
-    with pytest.raises(NotFoundException) as _:
-        gl_experimental.get_rule(rule.id)
 
 
 @pytest.mark.skip(reason="actions are global on account, the test matrix collides with itself")  # type: ignore
@@ -45,6 +42,13 @@ def test_create_action_with_human_review(gl_experimental: ExperimentalApi):
     )
     rule2 = gl_experimental.get_rule(rule.id)
     assert rule == rule2
+
+
+@pytest.mark.skip(reason="actions are global on account, the test matrix collides with itself")  # type: ignore
+def test_delete_action(gl_experimental: ExperimentalApi):
+    name = f"Test {datetime.utcnow()}"
+    det = gl_experimental.get_or_create_detector(name, "test_query")
+    rule = gl_experimental.create_rule(det, f"test_rule_{name}", "EMAIL", "[email protected]")
     gl_experimental.delete_rule(rule.id)
     with pytest.raises(NotFoundException) as _:
         gl_experimental.get_rule(rule.id)

test/unit/test_detector_reset.py

@@ -6,6 +6,7 @@
 from groundlight_openapi_client.exceptions import NotFoundException
 
 
+@pytest.mark.skip(reason="This is an expensive test, reset may take some time")
 def test_reset_retry(gl_experimental: ExperimentalApi):
     # Reset the detector, retrying in case the reset is still ongoing
     det = gl_experimental.create_detector(f"Test {datetime.utcnow()}", "test_query")
@@ -25,6 +26,7 @@ def test_reset_retry(gl_experimental: ExperimentalApi):
         raise Exception("Failed to reset detector")
 
 
+@pytest.mark.skip(reason="This test does not work with strong 0 shot models, enabled by default based on your account")
 def test_reset_training(gl_experimental: ExperimentalApi):
     # If we reset a detector, we should have low confidence after the reset
     low_confidence_threshold = 0.6

test/unit/test_experimental.py

@@ -56,6 +56,12 @@ def test_counting_detector(gl_experimental: ExperimentalApi):
     assert count_iq.result.count is not None
 
 
+@pytest.mark.skip(
+    reason=(
+        "General users currently currently can't use multiclass detectors. If you have questions, reach out"
+        " to Groundlight support, or upgrade your plan."
+    )
+)
 def test_multiclass_detector(gl_experimental: ExperimentalApi):
     """
     verify that we can create and submit to a multi-class detector