From be5b1f510b55fb53f80bac3b64948e6c5a748b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Fri, 4 Jul 2025 14:25:43 +0200 Subject: [PATCH 01/19] Don't call setup.py for building docs Calling setup.py from command line is deprecated by Python for a very long time now. --- .github/workflows/docs-pages.yaml | 2 -- .github/workflows/docs-pr.yaml | 2 -- docs/poetry.lock | 37 ++++++++++++++++++++++++------- docs/pyproject.toml | 5 +++-- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docs-pages.yaml b/.github/workflows/docs-pages.yaml index 4ff1f18d54..31f8dc74c5 100644 --- a/.github/workflows/docs-pages.yaml +++ b/.github/workflows/docs-pages.yaml @@ -27,8 +27,6 @@ jobs: python-version: '3.10' - name: Set up env run: make -C docs setupenv - - name: Build driver - run: python setup.py develop - name: Build docs run: make -C docs multiversion - name: Build redirects diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index dd9f9de66e..28a74f2e58 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -26,7 +26,5 @@ jobs: python-version: '3.10' - name: Set up env run: make -C docs setupenv - - name: Build driver - run: python setup.py develop - name: Build docs run: make -C docs test diff --git a/docs/poetry.lock b/docs/poetry.lock index bd62e7b50a..e59c4679a3 100644 --- a/docs/poetry.lock +++ b/docs/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "aenum" @@ -422,19 +422,18 @@ files = [ [[package]] name = "geomet" -version = "0.2.1.post1" -description = "GeoJSON <-> WKT/WKB conversion utilities" +version = "1.1.0" +description = "Pure Python conversion library for common geospatial data formats" optional = false -python-versions = ">2.6, !=3.3.*, <4" +python-versions = ">=3.7" groups = ["main"] files = [ - {file = "geomet-0.2.1.post1-py3-none-any.whl", hash = "sha256:a41a1e336b381416d6cbed7f1745c848e91defaa4d4c1bdc1312732e46ffad2b"}, - {file = "geomet-0.2.1.post1.tar.gz", hash = "sha256:91d754f7c298cbfcabd3befdb69c641c27fe75e808b27aa55028605761d17e95"}, + {file = "geomet-1.1.0-py3-none-any.whl", hash = "sha256:4372fe4e286a34acc6f2e9308284850bd8c4aa5bc12065e2abbd4995900db12f"}, + {file = "geomet-1.1.0.tar.gz", hash = "sha256:51e92231a0ef6aaa63ac20c443377ba78a303fd2ecd179dc3567de79f3c11605"}, ] [package.dependencies] click = "*" -six = "*" [[package]] name = "gevent" @@ -1010,6 +1009,28 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "scylla-driver" +version = "3.29.3" +description = "Scylla Driver for Apache Cassandra" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [] +develop = true + +[package.dependencies] +geomet = ">=1.1" +pyyaml = ">5.0" + +[package.extras] +cle = ["cryptography (>=35.0)"] +graph = ["gremlinpython (==3.4.6)"] + +[package.source] +type = "directory" +url = ".." + [[package]] name = "setuptools" version = "78.1.1" @@ -1831,4 +1852,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "302d62881c3c0d5ae60560928810117c52594d173faf903ba5d3cfeb49554dd3" +content-hash = "432ab6b3744422cd7526b9d863d4f2de57fec071049ecde4b690e15b6dd36551" diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 205b142c76..3d178f1936 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -8,13 +8,12 @@ package-mode = false [tool.poetry.dependencies] eventlet = "^0.33.3" futures = "2.2.0" -geomet = ">=0.1,<0.3" gevent = "^23.9.1" gremlinpython = "3.4.7" python = "^3.10" pygments = "^2.18.0" recommonmark = "0.7.1" -redirects_cli ="~0.1.2" +redirects_cli = "~0.1.2" sphinx-autobuild = "^2024.4.19" sphinx-sitemap = "^2.6.0" sphinx-scylladb-theme = "^1.8.1" @@ -23,6 +22,8 @@ Sphinx = "^7.3.7" scales = "^1.0.9" six = ">=1.9" tornado = ">=4.0,<5.0" +scylla-driver = { path = "../", develop = true } + [build-system] requires = ["poetry>=1.8.0"] From 9aa12494c9f82e8e17e20b85d8287a6cf920ef67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 22 May 2025 15:17:06 +0200 Subject: [PATCH 02/19] Format pyproject.toml --- pyproject.toml | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f3069869b4..c39452a4eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,7 @@ [project] name = "scylla-driver" description = "Scylla Driver for Apache Cassandra" -authors = [ - {name = "ScyllaDB"}, -] +authors = [{ name = "ScyllaDB" }] keywords = ["cassandra", "cql", "orm", "dse", "graph"] classifiers = [ 'Development Status :: 5 - Production/Stable', @@ -19,12 +17,9 @@ classifiers = [ 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Software Development :: Libraries :: Python Modules' -] -dependencies = [ - 'geomet>=1.1', - 'pyyaml > 5.0' + 'Topic :: Software Development :: Libraries :: Python Modules', ] +dependencies = ['geomet>=1.1', 'pyyaml > 5.0'] dynamic = ["version", "readme"] [project.urls] @@ -34,7 +29,7 @@ dynamic = ["version", "readme"] "Issues" = "https://github.com/scylladb/python-driver/issues" [project.optional-dependencies] -graph = ['gremlinpython==3.4.6'] +graph = ['gremlinpython==3.4.6'] cle = ['cryptography>=35.0'] test = [ "pytest", @@ -56,22 +51,26 @@ test = [ [tool.setuptools] include-package-data = true -packages=[ - 'cassandra', 'cassandra.io', 'cassandra.cqlengine', 'cassandra.graph', - 'cassandra.datastax', 'cassandra.datastax.insights', 'cassandra.datastax.graph', - 'cassandra.datastax.graph.fluent', 'cassandra.datastax.cloud', 'cassandra.scylla', - 'cassandra.column_encryption' +packages = [ + 'cassandra', + 'cassandra.io', + 'cassandra.cqlengine', + 'cassandra.graph', + 'cassandra.datastax', + 'cassandra.datastax.insights', + 'cassandra.datastax.graph', + 'cassandra.datastax.graph.fluent', + 'cassandra.datastax.cloud', + 'cassandra.scylla', + 'cassandra.column_encryption', ] [tool.setuptools.dynamic] -version = {attr = "cassandra.__version__"} # any module attribute compatible with ast.literal_eval -readme = {file = "README.rst", content-type = "text/x-rst"} +version = { attr = "cassandra.__version__" } # any module attribute compatible with ast.literal_eval +readme = { file = "README.rst", content-type = "text/x-rst" } [build-system] -requires = [ - "setuptools>=42", - "Cython", -] +requires = ["setuptools>=42", "Cython"] build-backend = "setuptools.build_meta" @@ -84,7 +83,17 @@ tag_regex = '(?P\d*?\.\d*?\.\d*?)-scylla' [tool.cibuildwheel] build-frontend = "build[uv]" environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "yes", CFLAGS = "-g0 -O3" } -skip = ["cp2*", "cp36*", "pp36*", "cp37*", "pp37*", "cp38*", "pp38*", "*i686", "*musllinux*"] +skip = [ + "cp2*", + "cp36*", + "pp36*", + "cp37*", + "pp37*", + "cp38*", + "pp38*", + "*i686", + "*musllinux*", +] build = ["cp3*", "pp3*"] before-test = "pip install -r {project}/test-requirements.txt" @@ -98,24 +107,24 @@ manylinux-pypy_aarch64-image = "manylinux_2_28" before-build = "rm -rf ~/.pyxbld && rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux && yum install -y libffi-devel libev libev-devel openssl openssl-devel" test-command = [ - "pytest {package}/tests/unit", - "EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py", + "pytest {package}/tests/unit", + "EVENT_LOOP_MANAGER=gevent pytest {package}/tests/unit/io/test_geventreactor.py", ] [tool.cibuildwheel.macos] build-frontend = "build" test-command = [ - "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'" + "pytest {project}/tests/unit -k 'not (test_multi_timer_validation or test_empty_connections or test_timer_cancellation)'", ] [tool.cibuildwheel.windows] build-frontend = "build" test-command = [ - "pytest {project}/tests/unit -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"" + "pytest {project}/tests/unit -k \"not (test_deserialize_date_range_year or test_datetype or test_libevreactor)\"", ] # TODO: set CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST to yes when https://github.com/scylladb/python-driver/issues/429 is fixed -environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no"} +environment = { CASS_DRIVER_BUILD_CONCURRENCY = "2", CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = "no" } [[tool.cibuildwheel.overrides]] select = "pp*" From 493e7e11cbb5a9a1eeab1524bf0ba4a4ff47b2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 22 May 2025 15:17:29 +0200 Subject: [PATCH 03/19] pyproject.toml: Use license field --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c39452a4eb..01a3e20817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,6 @@ keywords = ["cassandra", "cql", "orm", "dse", "graph"] classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', @@ -21,6 +20,7 @@ classifiers = [ ] dependencies = ['geomet>=1.1', 'pyyaml > 5.0'] dynamic = ["version", "readme"] +license = "Apache-2.0" [project.urls] "Homepage" = "https://github.com/scylladb/python-driver" From 11207ba8066f9fe82c644e1f6d812903ec3308d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 22 May 2025 15:18:05 +0200 Subject: [PATCH 04/19] pyproject.toml: Specify required Python version --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 01a3e20817..502935f13c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ dependencies = ['geomet>=1.1', 'pyyaml > 5.0'] dynamic = ["version", "readme"] license = "Apache-2.0" +requires-python = ">=3.9" [project.urls] "Homepage" = "https://github.com/scylladb/python-driver" From e2d889c0421c1d8a13ef44c531f333bb2a716950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 22 May 2025 15:18:38 +0200 Subject: [PATCH 05/19] pyproject.toml: Move to dependency-groups This is a new standardized way to have dev dependencies. --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 502935f13c..75f8add818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,9 @@ requires-python = ">=3.9" [project.optional-dependencies] graph = ['gremlinpython==3.4.6'] cle = ['cryptography>=35.0'] -test = [ + +[dependency-groups] +dev = [ "pytest", "PyYAML", "pytz", From 344041b5af7a07d787d8f9d0bcfb654461cece8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 17:46:07 +0200 Subject: [PATCH 06/19] pytest.ini: Add -rf flag We use this in CI, and it is generally useful, so lets not require manually adding it. --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 0846273427..d01008d3a1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ log_format = %(asctime)s.%(msecs)03d %(levelname)s [%(module)s:%(lineno)s]: %(me log_level = DEBUG log_date_format = %Y-%m-%d %H:%M:%S xfail_strict=true +addopts = -rf From da80ffdd42eef0839b7a5c12be899a4143a57f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 17:55:20 +0200 Subject: [PATCH 07/19] pytest.ini: Move to pyproject.toml This reduces amount of files in root and move towards centralizing configuration. --- pyproject.toml | 7 +++++++ pytest.ini | 6 ------ 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 75f8add818..ccfc07a98c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,13 @@ requires = ["setuptools>=42", "Cython"] build-backend = "setuptools.build_meta" +[tool.pytest.ini_options] +log_format = "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s:%(lineno)s]: %(message)s" +log_level = "DEBUG" +log_date_format = "%Y-%m-%d %H:%M:%S" +xfail_strict = true +addopts = "-rf" + [tool.setuptools_scm] version_file = "cassandra/_version.py" tag_regex = '(?P\d*?\.\d*?\.\d*?)-scylla' diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index d01008d3a1..0000000000 --- a/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -log_format = %(asctime)s.%(msecs)03d %(levelname)s [%(module)s:%(lineno)s]: %(message)s -log_level = DEBUG -log_date_format = %Y-%m-%d %H:%M:%S -xfail_strict=true -addopts = -rf From 407098c1a6755a76f5c968d443a2799d60bec3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Tue, 17 Jun 2025 19:48:37 +0200 Subject: [PATCH 08/19] Add ccm to dev deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ccfc07a98c..9b2c8de9ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev = [ "futurist", "asynctest", "pyyaml", + "ccm @ git+https://git@github.com/scylladb/scylla-ccm.git@master", ] [tool.setuptools] From 35c2436b685b463370323a9feeda704bea6f849f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Fri, 4 Jul 2025 11:23:04 +0200 Subject: [PATCH 09/19] dev dependencies: Remove Python version requirements Those were removed some time ago in test-requirements.txt, but not here. --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b2c8de9ae..c4535936c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,9 @@ dev = [ "scales", "pure-sasl", "twisted[tls]", - "gevent>=1.0; python_version < '3.13' and platform_machine != 'i686' and platform_machine != 'win32'", - "gevent==23.9.0; python_version < '3.13' and (platform_machine == 'i686' or platform_machine == 'win32')", - "eventlet>=0.33.3; python_version < '3.13'", + "gevent>=1.0; platform_machine != 'i686' and platform_machine != 'win32'", + "gevent==23.9.0; platform_machine == 'i686' or platform_machine == 'win32'", + "eventlet>=0.33.3", "cython", "packaging", "futurist", From dcdda0c8e6771fde77bfc0eadfd1574f486e66de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Fri, 4 Jul 2025 11:30:51 +0200 Subject: [PATCH 10/19] Remove platform gevent requirements They make it impossible to resolve dependencies on Python 3.13 because gevent 23.9.0 doesn't have wheels for it. This special case was, as far as I can tell, somehow ignored in our CI. By this I mean that even on windows build it used newer gevent. Because of that, removing those cases shouldn't break anything. --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c4535936c9..6d34b6566c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,7 @@ dev = [ "scales", "pure-sasl", "twisted[tls]", - "gevent>=1.0; platform_machine != 'i686' and platform_machine != 'win32'", - "gevent==23.9.0; platform_machine == 'i686' or platform_machine == 'win32'", + "gevent", "eventlet>=0.33.3", "cython", "packaging", From 2dd4737c5a7b934cba67888db4a0783bdfcfe758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 17:30:38 +0200 Subject: [PATCH 11/19] Move tablets test to standard tests Tablets are now available on Scylla version that we run CI with, so I see no reason to keep it separate. --- .github/workflows/integration-tests.yml | 6 ------ tests/integration/{experiments => standard}/test_tablets.py | 0 2 files changed, 6 deletions(-) rename tests/integration/{experiments => standard}/test_tablets.py (100%) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 65c3773648..706bc0f101 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -55,9 +55,3 @@ jobs: export EVENT_LOOP_MANAGER=${{ matrix.event_loop_manager }} export SCYLLA_VERSION='release:6.2' ./scripts/run_integration_test.sh tests/integration/standard/ tests/integration/cqlengine/ - - - name: Test tablets - run: | - export EVENT_LOOP_MANAGER=${{ matrix.event_loop_manager }} - export SCYLLA_VERSION='release:6.2' - ./scripts/run_integration_test.sh tests/integration/experiments/ diff --git a/tests/integration/experiments/test_tablets.py b/tests/integration/standard/test_tablets.py similarity index 100% rename from tests/integration/experiments/test_tablets.py rename to tests/integration/standard/test_tablets.py From c0ea18b7e2b29ef6ac9e4bbab778ef9076c69e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 17:50:26 +0200 Subject: [PATCH 12/19] integration-tests.yml: Move to uv --- .github/workflows/integration-tests.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 706bc0f101..bccbdc63cc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -44,14 +44,30 @@ jobs: java-version: ${{ matrix.java-version }} distribution: 'adopt' - - uses: actions/setup-python@v5 - name: Install Python ${{ matrix.python-version }} + - name: Install libev + run: sudo apt-get install libev4 libev-dev + + - name: Install uv + uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true + + # This is to get honest accounting of test time vs download time vs build time. + # Not strictly necessary for running tests. + - name: Build driver + run: uv sync + + # This is to get honest accounting of test time vs download time vs build time. + # Not strictly necessary for running tests. + - name: Download Scylla + run: | + export SCYLLA_VERSION='release:6.2' + uv run ccm create scylla-driver-temp -n 1 --scylla --version ${SCYLLA_VERSION} + uv run ccm remove - name: Test with pytest run: | export EVENT_LOOP_MANAGER=${{ matrix.event_loop_manager }} export SCYLLA_VERSION='release:6.2' - ./scripts/run_integration_test.sh tests/integration/standard/ tests/integration/cqlengine/ + export PROTOCOL_VERSION=4 + uv run pytest tests/integration/standard/ tests/integration/cqlengine/ From a94716e08f664b38a512cd68f84cbe203a0d3ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 17:52:01 +0200 Subject: [PATCH 13/19] Remove run_integration_test.sh No longer necessary for anything. --- scripts/run_integration_test.sh | 41 --------------------------------- 1 file changed, 41 deletions(-) delete mode 100755 scripts/run_integration_test.sh diff --git a/scripts/run_integration_test.sh b/scripts/run_integration_test.sh deleted file mode 100755 index 2944e045db..0000000000 --- a/scripts/run_integration_test.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/bash -e - -sudo apt-get install gcc python3-dev libev4 libev-dev - -aio_max_nr_recommended_value=1048576 -aio_max_nr=$(cat /proc/sys/fs/aio-max-nr) -echo "The current aio-max-nr value is $aio_max_nr" -if (( aio_max_nr != aio_max_nr_recommended_value )); then - sudo sh -c "echo 'fs.aio-max-nr = $aio_max_nr_recommended_value' >> /etc/sysctl.conf" - sudo sysctl -p /etc/sysctl.conf - echo "The aio-max-nr was changed from $aio_max_nr to $(cat /proc/sys/fs/aio-max-nr)" - if (( $(cat /proc/sys/fs/aio-max-nr) != aio_max_nr_recommended_value )); then - echo "The aio-max-nr value was not changed to $aio_max_nr_recommended_value" - exit 1 - fi -fi - -python3 -m venv .test-venv -source .test-venv/bin/activate -pip install --upgrade pip -pip install -U wheel setuptools - -# install driver wheel -pip install --ignore-installed -r test-requirements.txt pytest -pip install -e . - -# download awscli -pip install awscli - -# install scylla-ccm -pip install https://github.com/scylladb/scylla-ccm/archive/master.zip - -# download version - -ccm create scylla-driver-temp -n 1 --scylla --version ${SCYLLA_VERSION} -ccm remove - -# run test - -PROTOCOL_VERSION=4 pytest -rf $* - From 4b9f104515f746947c4dfb823a7e1442c0f5d0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 18:09:26 +0200 Subject: [PATCH 14/19] lib-build-and-push.yml: Move to uv --- .github/workflows/lib-build-and-push.yml | 26 ++++++++---------------- pyproject.toml | 3 +-- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/lib-build-and-push.yml b/.github/workflows/lib-build-and-push.yml index c911a8ffd5..3a1c9fa480 100644 --- a/.github/workflows/lib-build-and-push.yml +++ b/.github/workflows/lib-build-and-push.yml @@ -103,20 +103,14 @@ jobs: echo "CIBW_BEFORE_TEST=true" >> $GITHUB_ENV; echo "CIBW_BEFORE_TEST_WINDOWS=(exit 0)" >> $GITHUB_ENV; - - uses: actions/setup-python@v5 - name: Install Python + - name: Install uv + uses: astral-sh/setup-uv@v6 with: python-version: ${{ inputs.python-version }} - allow-prereleases: 'true' - - - name: Enable pip installing globally - if: runner.os == 'MacOs' || runner.os == 'Windows' - run: | - echo "PIP_BREAK_SYSTEM_PACKAGES=1" >> $GITHUB_ENV - name: Install cibuildwheel run: | - python3 -m pip install cibuildwheel==2.22.0 + uv tool install 'cibuildwheel==2.22.0' - name: Install OpenSSL for Windows if: runner.os == 'Windows' @@ -154,12 +148,12 @@ jobs: if: matrix.target != 'linux-aarch64' shell: bash run: | - GITHUB_WORKFLOW_REF="scylladb/python-driver/.github/workflows/lib-build-and-push.yml@refs/heads/master" python3 -m cibuildwheel --output-dir wheelhouse + GITHUB_WORKFLOW_REF="scylladb/python-driver/.github/workflows/lib-build-and-push.yml@refs/heads/master" cibuildwheel --output-dir wheelhouse - name: Build wheels for linux aarch64 if: matrix.target == 'linux-aarch64' run: | - GITHUB_WORKFLOW_REF="scylladb/python-driver/.github/workflows/lib-build-and-push.yml@refs/heads/master" CIBW_BUILD="cp3*" python -m cibuildwheel --archs aarch64 --output-dir wheelhouse + GITHUB_WORKFLOW_REF="scylladb/python-driver/.github/workflows/lib-build-and-push.yml@refs/heads/master" CIBW_BUILD="cp3*" cibuildwheel --archs aarch64 --output-dir wheelhouse - uses: actions/upload-artifact@v4 with: @@ -172,14 +166,12 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - name: Install Python + - name: Install uv + uses: astral-sh/setup-uv@v6 - name: Build sdist - run: | - pip install build - python -m build --sdist - + run: uv build --sdist + - uses: actions/upload-artifact@v4 with: name: source-dist diff --git a/pyproject.toml b/pyproject.toml index 6d34b6566c..aa163452f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,8 +105,7 @@ skip = [ "*musllinux*", ] build = ["cp3*", "pp3*"] - -before-test = "pip install -r {project}/test-requirements.txt" +test-groups = ["dev"] manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" From 6c88ff5b8881488c4621c52cfff5a80a83a98a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 18:11:02 +0200 Subject: [PATCH 15/19] Remove legacy dependencies files No longer needed for anything, and we should not have 2 places to keep dependencies in. It will, and already has, resulted in those places being out of sync. --- requirements.txt | 1 - test-requirements.txt | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 requirements.txt delete mode 100644 test-requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1d5f0bcfc4..0000000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -geomet>=1.1 diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 331094fab6..0000000000 --- a/test-requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ --r requirements.txt -scales -pytest -pytz -sure -pure-sasl -twisted[tls] -gevent>=1.0; platform_machine != 'i686' and platform_machine != 'win32' -gevent==23.9.0; platform_machine == 'i686' or platform_machine == 'win32' -eventlet>=0.33.3; -cython>=0.20,<0.30 -packaging -futurist -asynctest -pyyaml From 3fc08f3f79f02564c7c46599e0aa06890e04ce5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Thu, 3 Jul 2025 18:00:25 +0200 Subject: [PATCH 16/19] README-dev.rst: Recommend uv usage --- README-dev.rst | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/README-dev.rst b/README-dev.rst index f0c4815729..f158226de0 100644 --- a/README-dev.rst +++ b/README-dev.rst @@ -15,7 +15,7 @@ Releasing Building the Docs ================= -To build and preview the documentation for the ScyllaDB Python driver locally, you must first manually install `python-driver`. +To build and preview the documentation for the ScyllaDB Python driver locally, you must first manually install `python-driver`. This is necessary for autogenerating the reference documentation of the driver. You can find detailed instructions on how to install the driver in the `Installation guide `_. @@ -24,6 +24,14 @@ After installing the driver, you can build the documentation: - Install poetry: ``pip install poetry`` - To preview docs in your browser: ``make -C docs preview`` +Tooling +======= + +We recommend using `uv` tool for running tests, linters and basically everything else, +since it makes Python tooling ecosystem mostly usable. +To install it, see instructions at https://docs.astral.sh/uv/getting-started/installation/ +The rest of this document assumes you have `uv` installed. + Tests ===== @@ -31,13 +39,13 @@ Running Unit Tests ------------------ Unit tests can be run like so:: - python -m pytest tests/unit - EVENT_LOOP_MANAGER=gevent python -m pytest tests/unit/io/test_geventreactor.py - EVENT_LOOP_MANAGER=eventlet python -m pytest tests/unit/io/test_eventletreactor.py + uv run pytest tests/unit + EVENT_LOOP_MANAGER=gevent uv run pytest tests/unit/io/test_geventreactor.py + EVENT_LOOP_MANAGER=eventlet uv run pytest tests/unit/io/test_eventletreactor.py You can run a specific test method like so:: - python -m pytest tests/unit/test_connection.py::ConnectionTest::test_bad_protocol_version + uv run pytest tests/unit/test_connection.py::ConnectionTest::test_bad_protocol_version Running Integration Tests ------------------------- @@ -46,17 +54,17 @@ In order to run integration tests, you must specify a version to run using eithe * ``CASSANDRA_VERSION`` environment variable:: - SCYLLA_VERSION="release:5.1" python -m pytest tests/integration/standard tests/integration/cqlengine/ + SCYLLA_VERSION="release:5.1" uv run pytest tests/integration/standard tests/integration/cqlengine/ Or you can specify a scylla/cassandra directory (to test unreleased versions):: - SCYLLA_VERSION=/path/to/scylla pytest tests/integration/standard/ + SCYLLA_VERSION=/path/to/scylla uv run pytest tests/integration/standard/ Specifying the usage of an already running Scylla cluster ------------------------------------------------------------ The test will start the appropriate Scylla clusters when necessary but if you don't want this to happen because a Scylla cluster is already running the flag ``USE_CASS_EXTERNAL`` can be used, for example:: - USE_CASS_EXTERNAL=1 SCYLLA_VERSION='release:5.1' pytest tests/integration/standard + USE_CASS_EXTERNAL=1 SCYLLA_VERSION='release:5.1' uv run pytest tests/integration/standard Specify a Protocol Version for Tests ------------------------------------ @@ -66,30 +74,29 @@ The protocol version defaults to: - 5 for Cassandra >= 4.0, 4 for Cassandra >= 2.2, 3 for Cassandra >= 2.1, 2 for Cassandra >= 2.0 You can overwrite it with the ``PROTOCOL_VERSION`` environment variable:: - PROTOCOL_VERSION=3 SCYLLA_VERSION="release:5.1" python -m pytest tests/integration/standard tests/integration/cqlengine/ + PROTOCOL_VERSION=3 SCYLLA_VERSION="release:5.1" uv run pytest tests/integration/standard tests/integration/cqlengine/ Seeing Test Logs in Real Time ----------------------------- Sometimes it's useful to output logs for the tests as they run:: - python -m pytest -s tests/unit/ + uv run pytest -s tests/unit/ Use tee to capture logs and see them on your terminal:: - python -m pytest -s tests/unit/ 2>&1 | tee test.log + uv run pytest -s tests/unit/ 2>&1 | tee test.log Running the Benchmarks ====================== There needs to be a version of cassandra running locally so before running the benchmarks, if ccm is installed: - - ccm create benchmark_cluster -v 3.0.1 -n 1 -s + + uv run ccm create benchmark_cluster -v 3.0.1 -n 1 -s To run the benchmarks, pick one of the files under the ``benchmarks/`` dir and run it:: - python benchmarks/future_batches.py + uv run benchmarks/future_batches.py There are a few options. Use ``--help`` to see them all:: - python benchmarks/future_batches.py --help - + uv run benchmarks/future_batches.py --help From 8cd9c0412e0c26dd53573311eefc5d7e6e474e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Fri, 4 Jul 2025 12:25:52 +0200 Subject: [PATCH 17/19] setup.py: Build concurrently by default Because why not? It will be faster - on my machine ~4x faster. --- docs/installation.rst | 3 +++ setup.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index c8bf66dfdc..b57ad37f96 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -76,6 +76,9 @@ threads used to build the driver and any C extensions: $ # installing from pip $ CASS_DRIVER_BUILD_CONCURRENCY=8 pip install scylla-driver +Note that by default (when CASS_DRIVER_BUILD_CONCURRENCY is not specified), concurrency will be equal to the number of +logical cores on your machine. + OSX Installation Error ^^^^^^^^^^^^^^^^^^^^^^ If you're installing on OSX and have XCode 5.1 installed, you may see an error like this:: diff --git a/setup.py b/setup.py index be2c5ca816..340ded87ed 100644 --- a/setup.py +++ b/setup.py @@ -206,6 +206,9 @@ def eval_env_var_as_array(varname): sys.argv = [a for a in sys.argv if a not in ("--no-murmur3", "--no-libev", "--no-cython", "--no-extensions")] build_concurrency = int(os.environ.get('CASS_DRIVER_BUILD_CONCURRENCY', '0')) +if build_concurrency == 0: + build_concurrency = None + CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST = bool(os.environ.get('CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST', 'no') == 'yes') class NoPatchExtension(Extension): @@ -292,7 +295,7 @@ def run(self): def build_extensions(self): - if build_concurrency > 1: + if build_concurrency is None or build_concurrency > 1: self.check_extensions_list(self.extensions) import multiprocessing.pool @@ -431,7 +434,6 @@ def run_setup(extensions): setup(**kw) - run_setup(None) if has_cqlengine: From cf88b7edb373863b52ec0eee9744f701e3d78881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Fri, 4 Jul 2025 15:42:56 +0200 Subject: [PATCH 18/19] uv: Rebuild lib when cython stuff changes When a file or variable used by setup.py changes, we should rebuild the driver. If we don't we risk very unpleasant errors caused by source files and binary files differing. --- pyproject.toml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index aa163452f0..196a25b45c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,40 @@ requires = ["setuptools>=42", "Cython"] build-backend = "setuptools.build_meta" +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "setup.py" }, + # Cythonized modules + { file = "cassandra/io/libevwrapper.c" }, + { file = "cassandra/cmurmur3.c" }, + { file = "cassandra/cluster.py" }, + { file = "cassandra/concurrent.py" }, + { file = "cassandra/connection.py" }, + { file = "cassandra/cqltypes.py" }, + { file = "cassandra/metadata.py" }, + { file = "cassandra/pool.py" }, + { file = "cassandra/protocol.py" }, + { file = "cassandra/query.py" }, + { file = "cassandra/util.py" }, + { file = "cassandra/shard_info.py" }, + { file = "cassandra/*.pyx" }, + { file = "cassandra/*.pxd" }, + + # Env variables used in setup.py + { env = "CASS_DRIVER_LIBEV_INCLUDES" }, + { env = "CASS_DRIVER_LIBEV_LIBS" }, + { env = "CASS_DRIVER_NO_EXTENSIONS" }, + { env = "CASS_DRIVER_NO_LIBEV" }, + { env = "CASS_DRIVER_NO_CYTHON" }, + { env = "CASS_DRIVER_BUILD_CONCURRENCY" }, + { env = "CASS_DRIVER_BUILD_EXTENSIONS_ARE_MUST" }, + { env = "CASS_DRIVER_ALLOWED_CYTHON_VERSION" }, + + # used by setuptools_scm + { git = { commit = true, tags = true } }, +] + [tool.pytest.ini_options] log_format = "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s:%(lineno)s]: %(message)s" log_level = "DEBUG" From 1ad89022fdf93f4e21090d787ab6238654a2b7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Bary=C5=82a?= Date: Sun, 6 Jul 2025 16:02:11 +0200 Subject: [PATCH 19/19] build-sdist: Use input Python version --- .github/workflows/lib-build-and-push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lib-build-and-push.yml b/.github/workflows/lib-build-and-push.yml index 3a1c9fa480..b68ef4eba5 100644 --- a/.github/workflows/lib-build-and-push.yml +++ b/.github/workflows/lib-build-and-push.yml @@ -168,6 +168,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ inputs.python-version }} - name: Build sdist run: uv build --sdist