Fixing flacky privateca tests (#9075, #9081) (#9189)

* Fixing flacky privateca tests (#9075, #9081)

* Fixing backoff to be longer than 1 minute

Author: rsamborski

privateca/snippets/requirements-test.txt

@@ -1,4 +1,4 @@
 pytest==7.2.1
-google-auth==2.16.0
+google-auth==2.16.1
 cryptography==39.0.1
 backoff==2.2.1

privateca/snippets/requirements.txt

@@ -1,3 +1,2 @@
 google-cloud-private-ca==1.6.1
-google-cloud-kms==2.14.0
-google-cloud-monitoring==2.14.0
+google-cloud-monitoring==2.14.1

privateca/snippets/test_certificate_authorities.py

@@ -40,7 +40,14 @@ def generate_name() -> str:
     return "i" + uuid.uuid4().hex[:10]
 
 
-@backoff.on_exception(backoff.expo, Exception, max_tries=3)
+# We are hitting 5 CAs per minute limit which can't be changed
+# We set the backoff function to use 4 as base - this way the 3rd try
+# should wait for 64 seconds and avoid per minute quota
+def backoff_expo_wrapper():
+    return backoff.expo(base=4)
+
+
+@backoff.on_exception(backoff_expo_wrapper, Exception, max_tries=3)
 def test_create_certificate(capsys: typing.Any) -> None:
     CA_POOL_NAME = generate_name()
     CA_NAME = generate_name()
@@ -113,6 +120,7 @@ def test_update_certificate_authority(
     assert "Successfully updated the labels !" in out
 
 
+@backoff.on_exception(backoff_expo_wrapper, Exception, max_tries=3)
 def test_create_monitor_ca_policy(capsys: typing.Any) -> None:
     create_ca_monitor_policy(PROJECT)
 

privateca/snippets/test_subordinate_ca.py

@@ -38,7 +38,14 @@ def generate_name() -> str:
     return "test-" + uuid.uuid4().hex[:10]
 
 
-@backoff.on_exception(backoff.expo, Exception, max_tries=3)
+# We are hitting 5 CAs per minute limit which can't be changed
+# We set the backoff function to use 4 as base - this way the 3rd try
+# should wait for 64 seconds and avoid per minute quota
+def backoff_expo_wrapper():
+    return backoff.expo(base=4)
+
+
+@backoff.on_exception(backoff_expo_wrapper, Exception, max_tries=3)
 def test_subordinate_certificate_authority(
     certificate_authority, capsys: typing.Any
 ) -> None: