switch to setuptools_scm

.github/workflows/ci-tests.yml

@@ -249,6 +249,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: record cwltool version
+        run: pip install -U setuptools wheel && pip install setuptools_scm[toml] && python setup.py --version
       - name: build & test cwltool_module container
         run: ./build-cwltool-docker.sh
 

.gitignore

@@ -53,6 +53,8 @@ value
 
 .python-version
 
+cwltool/_version.py
+
 # Folder created when using make
 cwltool_deps
 docs/_build/

MANIFEST.in

@@ -2,7 +2,7 @@ include README.rst CODE_OF_CONDUCT.md CONTRIBUTING.md
 include MANIFEST.in
 include LICENSE.txt
 include *requirements.txt mypy.ini tox.ini
-include gittaggers.py Makefile cwltool.py
+include Makefile cwltool.py
 recursive-include mypy-stubs *.pyi *.py
 include tests/*
 include tests/cwl-conformance/cwltool-conftest.py

Makefile

@@ -77,7 +77,7 @@ check-python3:
 	python --version 2>&1 | grep "Python 3"
 
 dist/${MODULE}-$(VERSION).tar.gz: check-python3 $(SOURCES)
-	python setup.py sdist bdist_wheel
+	python -m build
 
 ## docs                   : make the docs
 docs: FORCE
@@ -122,10 +122,10 @@ codespell-fix:
 
 ## format                 : check/fix all code indentation and formatting (runs black)
 format:
-	black --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs
+	black --exclude cwltool/schemas --exclude cwltool/_version.py setup.py cwltool.py cwltool tests mypy-stubs
 
 format-check:
-	black --diff --check --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs
+	black --diff --check --exclude cwltool/schemas setup.py --exclude cwltool/_version.py cwltool.py cwltool tests mypy-stubs
 
 ## pylint                 : run static code analysis on Python code
 pylint: $(PYSOURCES)
@@ -202,11 +202,13 @@ release-test: check-python3 FORCE
 	./release-test.sh
 
 release: release-test
+	git tag ${VERSION}
 	. testenv2/bin/activate && \
-		python testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
+		pip install build && \
+		python -m build testenv2/src/${PACKAGE} && \
 		pip install twine && \
-		twine upload testenv2/src/${MODULE}/dist/* && \
-		git tag ${VERSION} && git push --tags
+		twine upload testenv2/src/${PACKAGE}/dist/* && \
+		git push --tags
 
 flake8: $(PYSOURCES)
 	flake8 $^

build-cwltool-docker.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 set -ex
-docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module .
-docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool .
+engine=${ENGINE:-docker}  # example: `ENGINE=podman ./build-cwltool-docker.sh`
+${engine} build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module .
+${engine} build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool .
 
-docker run -t -v /var/run/docker.sock:/var/run/docker.sock \
+${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \
 	-v /tmp:/tmp \
 	-v "$PWD":/tmp/cwltool \
 	quay.io/commonwl/cwltool_module /bin/sh -c \

cwltool.Dockerfile

@@ -4,7 +4,8 @@ RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linu
 
 WORKDIR /cwltool
 COPY . .
-RUN CWLTOOL_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad \
+RUN export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CWLTOOL=$(grep __version__ cwltool/_version.py  | awk -F\' '{ print $2 }') ; \
+	CWLTOOL_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad \
 	--wheel-dir=/wheels .[deps]  # --verbose
 RUN rm /wheels/schema_salad*
 RUN pip install "black~=22.0"

cwltool/cwlprov/ro.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 import tempfile
+import urllib
 import uuid
 from pathlib import Path, PurePosixPath
 from typing import (
@@ -429,7 +430,7 @@ def generate_snapshot(self, prov_dep: CWLObjectType) -> None:
         self.self_check()
         for key, value in prov_dep.items():
             if key == "location" and cast(str, value).split("/")[-1]:
-                location = cast(str, value)
+                location = urllib.parse.unquote(cast(str, value))
                 filename = location.split("/")[-1]
                 path = os.path.join(self.folder, SNAPSHOT, filename)
                 filepath = ""

docs/conf.py

@@ -6,22 +6,24 @@
 
 # -- Path setup --------------------------------------------------------------
 
+import importlib.metadata
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
 import sys
-from datetime import datetime
 import time
-import importlib.metadata
+from datetime import datetime, timezone
 
 sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------
 
-build_date = datetime.utcfromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH", time.time())))
+build_date = datetime.fromtimestamp(
+    int(os.environ.get("SOURCE_DATE_EPOCH", time.time())), timezone.utc
+)
 project = "Common Workflow Language reference implementation"
 copyright = f"2019 — {build_date.year}, Peter Amstutz and contributors to the CWL Project"
 author = "Peter Amstutz and Common Workflow Language Project contributors"

pyproject.toml

@@ -1,6 +1,7 @@
 [build-system]
 requires = [
     "setuptools>=45",
+    "setuptools_scm[toml]>=8.0.4,<9",
     "mypy==1.6.0",  # also update mypy-requirements.txt
     "types-requests",
     "types-psutil",
@@ -14,6 +15,9 @@ requires = [
 ]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools_scm]
+write_to = "cwltool/_version.py"
+
 [tool.black]
 line-length = 100
 target-version = [ "py38" ]

release-test.sh

@@ -42,7 +42,7 @@ then
 	rm -f testenv1/lib/python-wheels/setuptools* \
 		&& pip install --force-reinstall -U pip==${pipver} \
 		&& pip install setuptools==${setuptoolsver} wheel
-	pip install  --no-build-isolation -rtest-requirements.txt ".${extras}"
+	pip install -rtest-requirements.txt ".${extras}"
 	#make test
 	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 	# mkdir testenv1/not-${module}
@@ -68,7 +68,7 @@ rm -f lib/python-wheels/setuptools* \
 # The following can fail if you haven't pushed your commits to ${repo}
 pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}"
 pushd src/${package}
-pip install -rtest-requirements.txt
+pip install -rtest-requirements.txt build
 make dist
 #make test
 cp dist/${package}*tar.gz ../../../testenv3/
@@ -88,7 +88,7 @@ rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
 package_tar=$(find . -name "${package}*tar.gz")
-pip install "-r${DIR}/test-requirements.txt" udocker
+pip install "-r${DIR}/test-requirements.txt" udocker build
 pip install "${package_tar}${extras}"
 udocker install
 mkdir out
@@ -97,7 +97,7 @@ pushd out/${package}*
 make dist
 make test
 pip install "-r${DIR}/mypy-requirements.txt"
-make mypy
+make mypyc
 pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 mkdir ../not-${module}
 pushd ../not-${module}

setup.py

@@ -3,9 +3,7 @@
 import os
 import sys
 import warnings
-from typing import Type
 
-import setuptools.command.egg_info as egg_info_cmd
 from setuptools import setup
 
 if os.name == "nt":
@@ -25,13 +23,6 @@
 SETUP_DIR = os.path.dirname(__file__)
 README = os.path.join(SETUP_DIR, "README.rst")
 
-try:
-    import gittaggers
-
-    Tagger: Type[egg_info_cmd.egg_info] = gittaggers.EggInfoFromGit
-except ImportError:
-    Tagger = egg_info_cmd.egg_info
-
 NEEDS_PYTEST = {"pytest", "test", "ptr"}.intersection(sys.argv)
 PYTEST_RUNNER = ["pytest-runner", "pytest-cov"] if NEEDS_PYTEST else []
 USE_MYPYC = False
@@ -94,7 +85,6 @@
 
 setup(
     name="cwltool",
-    version="3.1",
     description="Common workflow language reference implementation",
     long_description=open(README).read(),
     long_description_content_type="text/x-rst",
@@ -130,7 +120,8 @@
         "deps": ["galaxy-tool-util >= 22.1.2, <24", "galaxy-util <24"],
     },
     python_requires=">=3.8, <4",
-    setup_requires=PYTEST_RUNNER,
+    use_scm_version=True,
+    setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"],
     test_suite="tests",
     tests_require=[
         "bagit >= 1.6.4, < 1.9",
@@ -142,7 +133,6 @@
     ],
     entry_points={"console_scripts": ["cwltool=cwltool.main:run"]},
     zip_safe=True,
-    cmdclass={"egg_info": Tagger},
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",

tests/test_examples.py

@@ -6,6 +6,7 @@
 import stat
 import subprocess
 import sys
+import urllib.parse
 from io import StringIO
 from pathlib import Path
 from typing import Any, Dict, List, Union, cast
@@ -1719,23 +1720,23 @@ def test_expression_tool_class() -> None:
     factory = cwltool.factory.Factory()
     tool_path = get_data("tests/wf/parseInt-tool.cwl")
     expression_tool = factory.make(tool_path).t
-    assert str(expression_tool) == f"ExpressionTool: file://{tool_path}"
+    assert urllib.parse.unquote(str(expression_tool)) == f"ExpressionTool: file://{tool_path}"
 
 
 def test_operation_class() -> None:
     """Confirm properties of the AbstractOperation class."""
     factory = cwltool.factory.Factory()
     tool_path = get_data("tests/wf/operation/abstract-cosifer.cwl")
     expression_tool = factory.make(tool_path).t
-    assert str(expression_tool) == f"AbstractOperation: file://{tool_path}"
+    assert urllib.parse.unquote(str(expression_tool)) == f"AbstractOperation: file://{tool_path}"
 
 
 def test_command_line_tool_class() -> None:
     """Confirm properties of the CommandLineTool class."""
     factory = cwltool.factory.Factory()
     tool_path = get_data("tests/echo.cwl")
     expression_tool = factory.make(tool_path).t
-    assert str(expression_tool) == f"CommandLineTool: file://{tool_path}"
+    assert urllib.parse.unquote(str(expression_tool)) == f"CommandLineTool: file://{tool_path}"
 
 
 def test_record_default_with_long(tmp_path: Path) -> None:

tests/test_load_tool.py

@@ -1,5 +1,6 @@
 """Tests for cwltool.load_tool."""
 import logging
+import urllib.parse
 from pathlib import Path
 
 import pytest
@@ -133,17 +134,17 @@ def test_import_tracked() -> None:
 
     loadingContext = LoadingContext({"fast_parser": True})
     tool = load_tool(get_data("tests/wf/811-12.cwl"), loadingContext)
-    path = "import:file://%s" % get_data("tests/wf/schemadef-type.yml")
+    path = f"import:file://{get_data('tests/wf/schemadef-type.yml')}"
+    path2 = f"import:file://{urllib.parse.quote(get_data('tests/wf/schemadef-type.yml'))}"
 
     assert tool.doc_loader is not None
-    assert path in tool.doc_loader.idx
+    assert path in tool.doc_loader.idx or path2 in tool.doc_loader.idx
 
     loadingContext = LoadingContext({"fast_parser": False})
     tool = load_tool(get_data("tests/wf/811.cwl"), loadingContext)
-    path = "import:file://%s" % get_data("tests/wf/schemadef-type.yml")
 
     assert tool.doc_loader is not None
-    assert path in tool.doc_loader.idx
+    assert path in tool.doc_loader.idx or path2 in tool.doc_loader.idx
 
 
 def test_load_badhints() -> None:

tests/test_mpi.py

@@ -6,8 +6,8 @@
 from pathlib import Path
 from typing import Any, Generator, List, MutableMapping, Optional, Tuple
 
-import pkg_resources
 import pytest
+from importlib_resources import files
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Names
 from schema_salad.utils import yaml_no_ts
@@ -281,12 +281,11 @@ def test_env_passing(monkeypatch: pytest.MonkeyPatch) -> None:
 # Reading the schema is super slow - cache for the session
 @pytest.fixture(scope="session")
 def schema_ext11() -> Generator[Names, None, None]:
-    with pkg_resources.resource_stream("cwltool", "extensions-v1.1.yml") as res:
-        ext11 = res.read().decode("utf-8")
-        cwltool.process.use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11)
-        schema = cwltool.process.get_schema("v1.1")[1]
-        assert isinstance(schema, Names)
-        yield schema
+    ext11 = files("cwltool").joinpath("extensions-v1.1.yml").read_text("utf-8")
+    cwltool.process.use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11)
+    schema = cwltool.process.get_schema("v1.1")[1]
+    assert isinstance(schema, Names)
+    yield schema
 
 
 mpiReq = CommentedMap({"class": MPIRequirementName, "processes": 1})

tests/test_path_checks.py

@@ -107,7 +107,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None:
     assert main(params) == 0
 
 
-class TestFsAccess(StdFsAccess):
+class StubFsAccess(StdFsAccess):
     """Stub fs access object that doesn't rely on the filesystem."""
 
     def glob(self, pattern: str) -> List[str]:
@@ -195,7 +195,7 @@ def test_clt_returns_specialchar_names(tmp_path: Path) -> None:
         builder.files, builder.stagedir, RuntimeContext(), True
     )
     builder.outdir = "/var/spool/cwl"
-    fs_access = TestFsAccess("")
+    fs_access = StubFsAccess("")
 
     result = cast(
         CWLObjectType,