tests/integration: introduced @xfail_scylla_version_lt

some test that was marked with @xfail_scylla, are now passing
with newer release, i.e. some issue were fixed.

so this new decorator can xfail up to a certion scylla_version

note that it support both a oss version, and enterprise version

Author: fruch

tests/integration/__init__.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import os
+from distutils.log import fatal
+
 from cassandra.cluster import Cluster
 
 from tests import connection_class, EVENT_LOOP_MANAGER
@@ -299,6 +301,8 @@ def get_unsupported_lower_protocol():
     This is used to determine the lowest protocol version that is NOT
     supported by the version of C* running
     """
+    if SCYLLA_VERSION is not None:
+        return 2
     if CASSANDRA_VERSION >= Version('3.0'):
         return 2
     else:
@@ -310,7 +314,8 @@ def get_unsupported_upper_protocol():
     This is used to determine the highest protocol version that is NOT
     supported by the version of C* running
     """
-
+    if SCYLLA_VERSION is not None:
+        return 5
     if CASSANDRA_VERSION >= Version('4.0-a'):
         if DSE_VERSION:
             return None
@@ -389,6 +394,64 @@ def _id_and_mark(f):
 requires_custom_payload = pytest.mark.skipif(SCYLLA_VERSION is not None or PROTOCOL_VERSION < 4,
                                             reason='Scylla does not support custom payloads. Cassandra requires native protocol v4.0+')
 xfail_scylla = lambda reason, *args, **kwargs: pytest.mark.xfail(SCYLLA_VERSION is not None, reason=reason, *args, **kwargs)
+
+
+def xfail_scylla_version_lt(reason, oss_scylla_version, ent_scylla_version, *args, **kwargs):
+    """
+    It is used to mark tests that are going to fail on certain scylla versions.
+
+    :param reason: message to fail test with
+    :param oss_scylla_version: str, oss version from which test supposed to succeed
+    :param ent_scylla_version: str|list<str>|tuple<str>, enterprise versions from which test supposed to succeed.
+        Should contain at least first major: `x.1.1` or `x.1` or `x`.
+        If current version is lower than `first major` it expects it to fail
+        For rest versions it looks for a version with same `major` and `minor` if records exists and `micro` is less,
+          it expects it to fail, otherwise expects it to succeed.
+    """
+    if not reason.startswith("scylladb/scylladb#"):
+        raise ValueError('reason should start with scylladb/scylladb#<issue-id> to reference issue in scylla repo')
+
+    if not isinstance(ent_scylla_version, (tuple, list)):
+        ent_scylla_version = (ent_scylla_version,)
+    first_major = None
+    ent_scylla_version = [Version(v) for v in ent_scylla_version]
+
+    # Enterprise releases are tricky, here are the rules:
+    # 1. If something is fixed in `a.b.c` it is also fixed in `a.b.d` where `c <= d`
+    # 2. If something is fixed in `a.1.1` it is also fixed in `c.d.f` where `Version(a.1.1)` <= `Version(c.d.f`)`
+    # To make version matching work properly of enterprise we need to have a first major version `x.1.1` where it was fixed.
+    for ent_v in ent_scylla_version:
+        if ent_v < Version("2018.1"):
+            raise ValueError("ent_scylla_version should contain only enterprise versions, example: 2024.1.1")
+        if ent_v.minor in (1,0) and ent_v.micro in (1,0):
+            first_major = Version(str(ent_v.major)+"0.0")
+
+    if not first_major:
+        raise ValueError("ent_scylla_version should contain first major LTS: `x.1.1`")
+
+    if SCYLLA_VERSION is None:
+        return pytest.mark.skipif(False, reason="It is just a NoOP Decor, should not skip anything")
+
+    current_version = Version(get_scylla_version(SCYLLA_VERSION) if SCYLLA_VERSION is not None else '0.0.0')
+    is_enterprise = current_version > Version("2018.1")
+    if not is_enterprise:
+        return pytest.mark.xfail(current_version < Version(oss_scylla_version),
+                                 reason=reason, *args, **kwargs)
+
+    if first_major < current_version:
+        # Expect test to fail if it is less then first major
+        return pytest.mark.xfail(False, reason=reason, *args, **kwargs)
+
+    for ent_v in ent_scylla_version:
+        # Expect test to fail if there is any record with same major and minor and micro is higher
+        if ent_v.major == current_version.major and ent_v.minor == current_version.minor:
+            return pytest.mark.xfail(current_version < ent_v, reason=reason, *args, **kwargs)
+
+    # Noop
+    # Expect test to complete successfully
+    return pytest.mark.xfail(True, reason=reason, *args, **kwargs)
+
+
 incorrect_test = lambda reason='This test seems to be incorrect and should be fixed', *args, **kwargs: pytest.mark.xfail(reason=reason, *args, **kwargs)
 
 pypy = unittest.skipUnless(platform.python_implementation() == "PyPy", "Test is skipped unless it's on PyPy")

tests/integration/standard/test_cluster.py

@@ -42,7 +42,7 @@
 from tests.integration import use_cluster, get_server_versions, CASSANDRA_VERSION, \
     execute_until_pass, execute_with_long_wait_retry, get_node, MockLoggingHandler, get_unsupported_lower_protocol, \
     get_unsupported_upper_protocol, lessthanprotocolv3, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, \
-    lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, xfail_scylla, incorrect_test
+    lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, incorrect_test
 from tests.integration.util import assert_quiescent_pool_state
 import sys
 
@@ -288,7 +288,6 @@ def test_protocol_negotiation(self):
 
         cluster.shutdown()
 
-    @xfail_scylla("Failing with scylla because there is option to create a cluster with 'lower bound' protocol")
     def test_invalid_protocol_negotation(self):
         """
         Test for protocol negotiation when explicit versions are set

tests/integration/standard/test_metadata.py

@@ -42,7 +42,7 @@
                                greaterthancass21, assert_startswith, greaterthanorequalcass40,
                                greaterthanorequaldse67, lessthancass40,
                                TestCluster, DSE_VERSION, requires_java_udf, requires_composite_type,
-                               requires_collection_indexes, SCYLLA_VERSION)
+                               requires_collection_indexes, SCYLLA_VERSION, xfail_scylla_version_lt)
 
 from tests.util import wait_until
 
@@ -504,7 +504,8 @@ def test_indexes(self):
         self.assertIn('CREATE INDEX e_index', statement)
 
     @greaterthancass21
-    @requires_collection_indexes
+    @xfail_scylla_version_lt('scylladb/scylladb#19278 - index functions are not included to DESCRIBE',
+                             oss_scylla_version='6.0.3', ent_scylla_version='2025.1')
     def test_collection_indexes(self):
 
         self.session.execute("CREATE TABLE %s.%s (a int PRIMARY KEY, b map<text, text>)"
@@ -1207,7 +1208,8 @@ def test_export_keyspace_schema_udts(self):
         cluster.shutdown()
 
     @greaterthancass21
-    @pytest.mark.xfail(reason='Column name in CREATE INDEX is not quoted. It\'s a bug in driver or in Scylla')
+    @xfail_scylla_version_lt(reason='scylladb/scylladb#10707 - Column name in CREATE INDEX is not quoted',
+                             oss_scylla_version="5.2", ent_scylla_version="2023.1")
     def test_case_sensitivity(self):
         """
         Test that names that need to be escaped in CREATE statements are