diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 82b8510ec..98566a0c0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -249,6 +249,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: record cwltool version + run: pip install -U setuptools wheel && pip install setuptools_scm[toml] && python setup.py --version - name: build & test cwltool_module container run: ./build-cwltool-docker.sh diff --git a/.gitignore b/.gitignore index 7a280f0df..5941627f8 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ value .python-version +cwltool/_version.py + # Folder created when using make cwltool_deps docs/_build/ diff --git a/MANIFEST.in b/MANIFEST.in index 6b533819d..92f65cfe5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ include README.rst CODE_OF_CONDUCT.md CONTRIBUTING.md include MANIFEST.in include LICENSE.txt include *requirements.txt mypy.ini tox.ini -include gittaggers.py Makefile cwltool.py +include Makefile cwltool.py recursive-include mypy-stubs *.pyi *.py include tests/* include tests/cwl-conformance/cwltool-conftest.py diff --git a/Makefile b/Makefile index 7ede5f2d3..75fbde472 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ check-python3: python --version 2>&1 | grep "Python 3" dist/${MODULE}-$(VERSION).tar.gz: check-python3 $(SOURCES) - python setup.py sdist bdist_wheel + python -m build ## docs : make the docs docs: FORCE @@ -122,10 +122,10 @@ codespell-fix: ## format : check/fix all code indentation and formatting (runs black) format: - black --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs + black --exclude cwltool/schemas --exclude cwltool/_version.py setup.py cwltool.py cwltool tests mypy-stubs format-check: - black --diff --check --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs + black --diff --check --exclude cwltool/schemas setup.py --exclude cwltool/_version.py cwltool.py cwltool tests mypy-stubs ## pylint : run static code analysis on Python code pylint: $(PYSOURCES) @@ -202,11 +202,13 @@ release-test: check-python3 FORCE ./release-test.sh release: release-test + git tag ${VERSION} . testenv2/bin/activate && \ - python testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \ + pip install build && \ + python -m build testenv2/src/${PACKAGE} && \ pip install twine && \ - twine upload testenv2/src/${MODULE}/dist/* && \ - git tag ${VERSION} && git push --tags + twine upload testenv2/src/${PACKAGE}/dist/* && \ + git push --tags flake8: $(PYSOURCES) flake8 $^ diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index 97910069a..a70fdf4df 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -1,9 +1,10 @@ #!/bin/bash set -ex -docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module . -docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . +engine=${ENGINE:-docker} # example: `ENGINE=podman ./build-cwltool-docker.sh` +${engine} build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module . +${engine} build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . -docker run -t -v /var/run/docker.sock:/var/run/docker.sock \ +${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ diff --git a/cwltool.Dockerfile b/cwltool.Dockerfile index 306c2997c..4fa76b126 100644 --- a/cwltool.Dockerfile +++ b/cwltool.Dockerfile @@ -4,7 +4,8 @@ RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linu WORKDIR /cwltool COPY . . -RUN CWLTOOL_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad \ +RUN export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CWLTOOL=$(grep __version__ cwltool/_version.py | awk -F\' '{ print $2 }') ; \ + CWLTOOL_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad \ --wheel-dir=/wheels .[deps] # --verbose RUN rm /wheels/schema_salad* RUN pip install "black~=22.0" diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 3bcc9fddf..c34e32082 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -5,6 +5,7 @@ import os import shutil import tempfile +import urllib import uuid from pathlib import Path, PurePosixPath from typing import ( @@ -429,7 +430,7 @@ def generate_snapshot(self, prov_dep: CWLObjectType) -> None: self.self_check() for key, value in prov_dep.items(): if key == "location" and cast(str, value).split("/")[-1]: - location = cast(str, value) + location = urllib.parse.unquote(cast(str, value)) filename = location.split("/")[-1] path = os.path.join(self.folder, SNAPSHOT, filename) filepath = "" diff --git a/docs/conf.py b/docs/conf.py index fbc089caa..6e04b5d64 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,22 +6,24 @@ # -- Path setup -------------------------------------------------------------- +import importlib.metadata # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys -from datetime import datetime import time -import importlib.metadata +from datetime import datetime, timezone sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- -build_date = datetime.utcfromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH", time.time()))) +build_date = datetime.fromtimestamp( + int(os.environ.get("SOURCE_DATE_EPOCH", time.time())), timezone.utc +) project = "Common Workflow Language reference implementation" copyright = f"2019 — {build_date.year}, Peter Amstutz and contributors to the CWL Project" author = "Peter Amstutz and Common Workflow Language Project contributors" diff --git a/gittaggers.py b/gittaggers.py deleted file mode 100644 index c25dbe1af..000000000 --- a/gittaggers.py +++ /dev/null @@ -1,42 +0,0 @@ -import subprocess -import sys -import time - -import importlib.metadata - -from typing import Any - -from setuptools.command.egg_info import egg_info - -SETUPTOOLS_VER = importlib.metadata.version("setuptools").split(".") - -RECENT_SETUPTOOLS = ( - int(SETUPTOOLS_VER[0]) > 40 - or (int(SETUPTOOLS_VER[0]) == 40 and int(SETUPTOOLS_VER[1]) > 0) - or (int(SETUPTOOLS_VER[0]) == 40 and int(SETUPTOOLS_VER[1]) == 0 and int(SETUPTOOLS_VER[2]) > 0) -) - - -class EggInfoFromGit(egg_info): - """Tag the build with git commit timestamp. - - If a build tag has already been set (e.g., "egg_info -b", building - from source package), leave it alone. - """ - - def git_timestamp_tag(self) -> str: - gitinfo = subprocess.check_output( - ["git", "log", "--first-parent", "--max-count=1", "--format=format:%ct", "."] - ).strip() - return time.strftime(".%Y%m%d%H%M%S", time.gmtime(int(gitinfo))) - - def tags(self) -> Any: - if self.tag_build is None: - try: - self.tag_build = self.git_timestamp_tag() - except subprocess.CalledProcessError: - pass - return egg_info.tags(self) # type: ignore[no-untyped-call] - - if RECENT_SETUPTOOLS: - vtags = property(tags) diff --git a/pyproject.toml b/pyproject.toml index 0593d123f..26251a48e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [build-system] requires = [ "setuptools>=45", + "setuptools_scm[toml]>=8.0.4,<9", "mypy==1.6.0", # also update mypy-requirements.txt "types-requests", "types-psutil", @@ -14,6 +15,9 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] +write_to = "cwltool/_version.py" + [tool.black] line-length = 100 target-version = [ "py38" ] diff --git a/release-test.sh b/release-test.sh index 4faf6a184..254be6271 100755 --- a/release-test.sh +++ b/release-test.sh @@ -42,7 +42,7 @@ then rm -f testenv1/lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel - pip install --no-build-isolation -rtest-requirements.txt ".${extras}" + pip install -rtest-requirements.txt ".${extras}" #make test pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install # mkdir testenv1/not-${module} @@ -68,7 +68,7 @@ rm -f lib/python-wheels/setuptools* \ # The following can fail if you haven't pushed your commits to ${repo} pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}" pushd src/${package} -pip install -rtest-requirements.txt +pip install -rtest-requirements.txt build make dist #make test cp dist/${package}*tar.gz ../../../testenv3/ @@ -88,7 +88,7 @@ rm -f lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel package_tar=$(find . -name "${package}*tar.gz") -pip install "-r${DIR}/test-requirements.txt" udocker +pip install "-r${DIR}/test-requirements.txt" udocker build pip install "${package_tar}${extras}" udocker install mkdir out @@ -97,7 +97,7 @@ pushd out/${package}* make dist make test pip install "-r${DIR}/mypy-requirements.txt" -make mypy +make mypyc pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install mkdir ../not-${module} pushd ../not-${module} diff --git a/setup.py b/setup.py index 9dbed21b2..7b02dc972 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,7 @@ import os import sys import warnings -from typing import Type -import setuptools.command.egg_info as egg_info_cmd from setuptools import setup if os.name == "nt": @@ -25,13 +23,6 @@ SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, "README.rst") -try: - import gittaggers - - Tagger: Type[egg_info_cmd.egg_info] = gittaggers.EggInfoFromGit -except ImportError: - Tagger = egg_info_cmd.egg_info - NEEDS_PYTEST = {"pytest", "test", "ptr"}.intersection(sys.argv) PYTEST_RUNNER = ["pytest-runner", "pytest-cov"] if NEEDS_PYTEST else [] USE_MYPYC = False @@ -94,7 +85,6 @@ setup( name="cwltool", - version="3.1", description="Common workflow language reference implementation", long_description=open(README).read(), long_description_content_type="text/x-rst", @@ -130,7 +120,8 @@ "deps": ["galaxy-tool-util >= 22.1.2, <24", "galaxy-util <24"], }, python_requires=">=3.8, <4", - setup_requires=PYTEST_RUNNER, + use_scm_version=True, + setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", tests_require=[ "bagit >= 1.6.4, < 1.9", @@ -142,7 +133,6 @@ ], entry_points={"console_scripts": ["cwltool=cwltool.main:run"]}, zip_safe=True, - cmdclass={"egg_info": Tagger}, classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", diff --git a/tests/reloc/dir1/foo b/tests/reloc/dir1/foo deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/reloc/dir2 b/tests/reloc/dir2 deleted file mode 120000 index df490f837..000000000 --- a/tests/reloc/dir2 +++ /dev/null @@ -1 +0,0 @@ -dir1 \ No newline at end of file diff --git a/tests/test_examples.py b/tests/test_examples.py index 9a5b5ada8..c16e41306 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -6,6 +6,7 @@ import stat import subprocess import sys +import urllib.parse from io import StringIO from pathlib import Path from typing import Any, Dict, List, Union, cast @@ -1719,7 +1720,7 @@ def test_expression_tool_class() -> None: factory = cwltool.factory.Factory() tool_path = get_data("tests/wf/parseInt-tool.cwl") expression_tool = factory.make(tool_path).t - assert str(expression_tool) == f"ExpressionTool: file://{tool_path}" + assert urllib.parse.unquote(str(expression_tool)) == f"ExpressionTool: file://{tool_path}" def test_operation_class() -> None: @@ -1727,7 +1728,7 @@ def test_operation_class() -> None: factory = cwltool.factory.Factory() tool_path = get_data("tests/wf/operation/abstract-cosifer.cwl") expression_tool = factory.make(tool_path).t - assert str(expression_tool) == f"AbstractOperation: file://{tool_path}" + assert urllib.parse.unquote(str(expression_tool)) == f"AbstractOperation: file://{tool_path}" def test_command_line_tool_class() -> None: @@ -1735,7 +1736,7 @@ def test_command_line_tool_class() -> None: factory = cwltool.factory.Factory() tool_path = get_data("tests/echo.cwl") expression_tool = factory.make(tool_path).t - assert str(expression_tool) == f"CommandLineTool: file://{tool_path}" + assert urllib.parse.unquote(str(expression_tool)) == f"CommandLineTool: file://{tool_path}" def test_record_default_with_long(tmp_path: Path) -> None: diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 3d0cba161..cf2cb620b 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -1,5 +1,6 @@ """Tests for cwltool.load_tool.""" import logging +import urllib.parse from pathlib import Path import pytest @@ -133,17 +134,17 @@ def test_import_tracked() -> None: loadingContext = LoadingContext({"fast_parser": True}) tool = load_tool(get_data("tests/wf/811-12.cwl"), loadingContext) - path = "import:file://%s" % get_data("tests/wf/schemadef-type.yml") + path = f"import:file://{get_data('tests/wf/schemadef-type.yml')}" + path2 = f"import:file://{urllib.parse.quote(get_data('tests/wf/schemadef-type.yml'))}" assert tool.doc_loader is not None - assert path in tool.doc_loader.idx + assert path in tool.doc_loader.idx or path2 in tool.doc_loader.idx loadingContext = LoadingContext({"fast_parser": False}) tool = load_tool(get_data("tests/wf/811.cwl"), loadingContext) - path = "import:file://%s" % get_data("tests/wf/schemadef-type.yml") assert tool.doc_loader is not None - assert path in tool.doc_loader.idx + assert path in tool.doc_loader.idx or path2 in tool.doc_loader.idx def test_load_badhints() -> None: diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 3e3b5d491..e90d5d642 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -6,8 +6,8 @@ from pathlib import Path from typing import Any, Generator, List, MutableMapping, Optional, Tuple -import pkg_resources import pytest +from importlib_resources import files from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names from schema_salad.utils import yaml_no_ts @@ -281,12 +281,11 @@ def test_env_passing(monkeypatch: pytest.MonkeyPatch) -> None: # Reading the schema is super slow - cache for the session @pytest.fixture(scope="session") def schema_ext11() -> Generator[Names, None, None]: - with pkg_resources.resource_stream("cwltool", "extensions-v1.1.yml") as res: - ext11 = res.read().decode("utf-8") - cwltool.process.use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11) - schema = cwltool.process.get_schema("v1.1")[1] - assert isinstance(schema, Names) - yield schema + ext11 = files("cwltool").joinpath("extensions-v1.1.yml").read_text("utf-8") + cwltool.process.use_custom_schema("v1.1", "http://commonwl.org/cwltool", ext11) + schema = cwltool.process.get_schema("v1.1")[1] + assert isinstance(schema, Names) + yield schema mpiReq = CommentedMap({"class": MPIRequirementName, "processes": 1}) diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 2ebda7fe3..018a92120 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -107,7 +107,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: assert main(params) == 0 -class TestFsAccess(StdFsAccess): +class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" def glob(self, pattern: str) -> List[str]: @@ -195,7 +195,7 @@ def test_clt_returns_specialchar_names(tmp_path: Path) -> None: builder.files, builder.stagedir, RuntimeContext(), True ) builder.outdir = "/var/spool/cwl" - fs_access = TestFsAccess("") + fs_access = StubFsAccess("") result = cast( CWLObjectType, diff --git a/tests/test_relocate.py b/tests/test_relocate.py index e3c7e59c8..81877c776 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,4 +1,6 @@ import json +import os +import shutil import sys from pathlib import Path @@ -56,15 +58,19 @@ def test_for_conflict_file_names_nodocker(tmp_path: Path) -> None: def test_relocate_symlinks(tmp_path: Path) -> None: + shutil.copy(get_data("tests/reloc/test.cwl"), tmp_path) + (tmp_path / "dir1").mkdir() + (tmp_path / "dir1" / "foo").touch() + os.symlink(tmp_path / "dir1", tmp_path / "dir2") assert ( main( [ "--debug", "--outdir", - get_data("tests/reloc") + "/dir2", - get_data("tests/reloc/test.cwl"), + str(tmp_path / "dir2"), + str(tmp_path / "test.cwl"), "--inp", - get_data("tests/reloc") + "/dir2", + str(tmp_path / "dir2"), ] ) == 0