Merge remote-tracking branch 'origin/main' into input_annotations

Author: mr-c

.github/workflows/ci-tests.yml

@@ -55,7 +55,7 @@ jobs:
         run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
           allow-prereleases: true
@@ -104,7 +104,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -140,7 +140,7 @@ jobs:
         run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -194,7 +194,7 @@ jobs:
         run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
           cache: pip
@@ -207,7 +207,7 @@ jobs:
           CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
         run: ./conformance-test.sh
       - name: Archive test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results
           path: |
@@ -230,7 +230,7 @@ jobs:
           sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
           cache: pip
@@ -272,7 +272,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
           cache: pip

.github/workflows/codeql-analysis.yml

@@ -27,9 +27,9 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: python
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

conformance-test.sh

@@ -76,7 +76,8 @@ else
 	venv "${TMP_DIR}/cwl-conformance-venv"
 	pip install -U setuptools wheel pip
 	pip uninstall -y cwltool
-	pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
+	pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt"
+	CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
 	pip install 'cwltest>=2.3' pytest-cov pytest-xdist
 fi
 

cwltool/command_line_tool.py

@@ -866,7 +866,7 @@ def calc_checksum(location: str) -> Optional[str]:
                         and "checksum" in e
                         and e["checksum"] != "sha1$hash"
                     ):
-                        return cast(Optional[str], e["checksum"])
+                        return cast(str, e["checksum"])
                 return None
 
             def remove_prefix(s: str, prefix: str) -> str:

cwltool/process.py

@@ -1342,10 +1342,15 @@ def compute_checksums(fs_access: StdFsAccess, fileobj: CWLObjectType) -> None:
     if "checksum" not in fileobj:
         checksum = hashlib.sha1()  # nosec
         location = cast(str, fileobj["location"])
-        with fs_access.open(location, "rb") as f:
-            contents = f.read(1024 * 1024)
-            while contents != b"":
-                checksum.update(contents)
+        if "contents" in fileobj:
+            contents = cast(str, fileobj["contents"]).encode("utf-8")
+            checksum.update(contents)
+            fileobj["size"] = len(contents)
+        else:
+            with fs_access.open(location, "rb") as f:
                 contents = f.read(1024 * 1024)
+                while contents != b"":
+                    checksum.update(contents)
+                    contents = f.read(1024 * 1024)
+            fileobj["size"] = fs_access.size(location)
         fileobj["checksum"] = "sha1$%s" % checksum.hexdigest()
-        fileobj["size"] = fs_access.size(location)

lint-requirements.txt

@@ -1,3 +1,3 @@
 flake8-bugbear<23.13
-black~=23.11
+black~=23.12
 codespell

setup.py

@@ -115,6 +115,7 @@
         "argcomplete",
         "pyparsing != 3.0.2",  # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319
         "cwl-utils >= 0.32",
+        "spython >= 0.3.0",
     ],
     extras_require={
         "deps": [

tests/test_examples.py

@@ -1325,6 +1325,28 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
     assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
 
 
+@pytest.mark.parametrize("factor", test_factors)
+def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None:
+    """Confirm that running a CLT with a default literal file with caching succeeds."""
+    test_file = "tests/wf/extract_region_specs.cwl"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(
+        [
+            "--out",
+            str(tmp_path / "out"),
+            "--cachedir",
+            cache_dir,
+            get_data(test_file),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+
 def test_write_summary(tmp_path: Path) -> None:
     """Test --write-summary."""
     commands = [

tests/wf/extract_region_specs.cwl

@@ -0,0 +1,21 @@
+{
+"cwlVersion": "v1.0",
+"class": "CommandLineTool",
+"inputs": [
+    {
+        "type": "File",
+        "default": {
+            "class": "File",
+            "basename": "extract_regions.py",
+            "contents": "#!/usr/bin/env python3\n\nfrom __future__ import print_function, division\nimport sys\n\ninput_filename = sys.argv[1]\nif len(sys.argv) == 3:\n    fuzz = int(sys.argv[2])\nelse:\n    fuzz = 0\ninput_file = open(input_filename)\n\ncount = 0\nfor line in input_file:\n    if not line.startswith(\">\"):\n        continue\n    count += 1\n    contig_regions_file = open(\"contig_regions{}.txt\".format(count), \"w\")\n    proteins_list_file = open(\"proteins{}.txt\".format(count), \"w\")\n    fields = line.split(\"|\")\n    protein_id = fields[0][1:]\n    contig_id = fields[1]\n    r_start = int(fields[6])\n    if r_start > fuzz:\n        r_start = r_start - fuzz\n    r_end = int(fields[7]) + fuzz\n    print(\"{}:{}-{}\".format(contig_id, r_start, r_end), file=contig_regions_file)\n    print(protein_id, file=proteins_list_file)\n    contig_regions_file.close()\n    proteins_list_file.close()\n"
+        },
+        "inputBinding": {
+            "position": 1
+        },
+        "id": "scripts"
+    }
+],
+"outputs": [
+],
+"baseCommand": "cat"
+}