required outputs with secondaryFiles fix for containers

Author: mr-c

cwltool/command_line_tool.py

@@ -1454,7 +1454,10 @@ def collect_output(
                                         continue
                                     if isinstance(sfitem, str):
                                         sfitem = {"path": pathprefix + sfitem}
-                                    if not fs_access.exists(sfitem["path"]) and sf_required:
+                                    if (
+                                        not fs_access.exists(revmap(sfitem)["location"])
+                                        and sf_required
+                                    ):
                                         raise WorkflowException(
                                             "Missing required secondary file '%s'"
                                             % (sfitem["path"])

tests/secondary-files-required-container.cwl

@@ -0,0 +1,24 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.2
+class: CommandLineTool
+hints:
+  DockerRequirement:
+    dockerPull: docker.io/alpine:latest
+
+baseCommand:
+    - sh
+    - -c
+    - touch file.ext1 && touch file.ext2
+
+inputs: []
+
+outputs:
+  output:
+    type: File
+    secondaryFiles:
+      - pattern: ^.ext2
+        required: true
+    outputBinding:
+      glob: file.ext1
+

tests/test_docker.py

@@ -1,11 +1,18 @@
 """Tests for docker engine."""
+import json
 import re
 from pathlib import Path
 from shutil import which
 
 from cwltool.main import main
 
-from .util import get_data, get_main_output, needs_docker
+from .util import (
+    get_data,
+    get_main_output,
+    needs_docker,
+    needs_podman,
+    needs_singularity,
+)
 
 
 @needs_docker
@@ -136,3 +143,53 @@ def test_docker_strict_memory_limit_warning(tmp_path: Path) -> None:
     stderr = re.sub(r"\s\s+", " ", stderr)
     assert result_code == 0
     assert "Skipping Docker software container '--memory' limit" in stderr
+
+
+@needs_docker
+def test_docker_required_secfile(tmp_path: Path) -> None:
+    result_code, stdout, stderr = get_main_output(
+        [
+            "--outdir",
+            str(tmp_path),
+            get_data("tests/secondary-files-required-container.cwl"),
+        ]
+    )
+    assert result_code == 0, stderr
+    assert (
+        json.loads(stdout)["output"]["secondaryFiles"][0]["checksum"]
+        == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    )
+
+
+@needs_podman
+def test_podman_required_secfile(tmp_path: Path) -> None:
+    result_code, stdout, stderr = get_main_output(
+        [
+            "--outdir",
+            "--podman",
+            str(tmp_path),
+            get_data("tests/secondary-files-required-container.cwl"),
+        ]
+    )
+    assert result_code == 0, stderr
+    assert (
+        json.loads(stdout)["output"]["secondaryFiles"][0]["checksum"]
+        == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    )
+
+
+@needs_singularity
+def test_singularity_required_secfile(tmp_path: Path) -> None:
+    result_code, stdout, stderr = get_main_output(
+        [
+            "--outdir",
+            "--singularity",
+            str(tmp_path),
+            get_data("tests/secondary-files-required-container.cwl"),
+        ]
+    )
+    assert result_code == 0, stderr
+    assert (
+        json.loads(stdout)["output"]["secondaryFiles"][0]["checksum"]
+        == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    )