Merge branch 'main' into cuda-request-gpu-device-expr

Author: tetron

cwltool/argparser.py

@@ -478,14 +478,18 @@ def arg_parser() -> argparse.ArgumentParser:
     conda_dependencies = argparse.SUPPRESS
 
     if SOFTWARE_REQUIREMENTS_ENABLED:
-        dependency_resolvers_configuration_help = "Dependency resolver "
-        "configuration file describing how to adapt 'SoftwareRequirement' "
-        "packages to current system."
+        dependency_resolvers_configuration_help = (
+            "Dependency resolver "
+            "configuration file describing how to adapt 'SoftwareRequirement' "
+            "packages to current system."
+        )
         dependencies_directory_help = (
-            "Defaut root directory used by dependency resolvers configuration."
+            "Default root directory used by dependency resolvers configuration."
+        )
+        use_biocontainers_help = (
+            "Use biocontainers for tools without an "
+            "explicitly annotated Docker container."
         )
-        use_biocontainers_help = "Use biocontainers for tools without an "
-        "explicitly annotated Docker container."
         conda_dependencies = (
             "Short cut to use Conda to resolve 'SoftwareRequirement' packages."
         )
@@ -869,12 +873,24 @@ def add_argument(
             fieldname = name + "." + shortname(field["name"])
             fieldtype = field["type"]
             fielddescription = field.get("doc", "")
-            add_argument(toolparser, fieldname, fieldtype, records, fielddescription)
+            add_argument(
+                toolparser,
+                fieldname,
+                fieldtype,
+                records,
+                fielddescription,
+                default=default.get(shortname(field["name"]), None)
+                if default
+                else None,
+                input_required=required,
+            )
         return
     elif inptype == "string":
         atype = str
     elif inptype == "int":
         atype = int
+    elif inptype == "long":
+        atype = int
     elif inptype == "double":
         atype = float
     elif inptype == "float":

cwltool/checker.py

@@ -389,6 +389,11 @@ def check_all_types(
                         )
             else:
                 parm_id = cast(str, sink[sourceField])
+                if parm_id not in src_dict:
+                    raise SourceLine(sink, sourceField, ValidationException).makeError(
+                        f"{sourceField} not found: {parm_id}"
+                    )
+
                 srcs_of_sink = [src_dict[parm_id]]
                 linkMerge = None
 

cwltool/command_line_tool.py

@@ -1581,7 +1581,8 @@ def collect_output(
                     return None
 
         if (
-            not empty_and_optional
+            not result
+            and not empty_and_optional
             and isinstance(schema["type"], MutableMapping)
             and schema["type"]["type"] == "record"
         ):

cwltool/main.py

@@ -1303,7 +1303,6 @@ def main(
                 tool, job_order_object = tool.result(tfjob_order, tfout, runtimeContext)
                 if not job_order_object:
                     job_order_object = None
-
             try:
                 initialized_job_order_object = init_job_order(
                     job_order_object,

tests/checker_wf/broken-wf4.cwl

@@ -0,0 +1,20 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+inputs:
+  letters0:
+    type: string
+    default: "a0"
+
+outputs:
+  all:
+    type: File
+    outputSource: echo_w
+
+steps:
+  echo_w:
+    run: echo.cwl
+    in:
+      echo_in: letters0
+    out: [txt]
+

tests/test_examples.py

@@ -1685,3 +1685,59 @@ def test_command_line_tool_class() -> None:
     tool_path = get_data("tests/echo.cwl")
     expression_tool = factory.make(tool_path).t
     assert str(expression_tool) == f"CommandLineTool: file://{tool_path}"
+
+
+def test_record_default_with_long() -> None:
+    """Confirm that record defaults are respected."""
+    tool_path = get_data("tests/wf/paramref_arguments_roundtrip.cwl")
+    err_code, stdout, stderr = get_main_output([tool_path])
+    assert err_code == 0
+    result = json.loads(stdout)["same_record"]
+    assert result["first"] == "y"
+    assert result["second"] == 23
+    assert result["third"] == 2.3
+    assert result["fourth"] == 4242424242
+    assert result["fifth"] == 4200000000000000000000000000000000000000000
+    assert result["sixth"]["class"] == "File"
+    assert result["sixth"]["basename"] == "whale.txt"
+    assert result["sixth"]["size"] == 1111
+    assert (
+        result["sixth"]["checksum"] == "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376"
+    )
+
+
+def test_record_outputeval() -> None:
+    """Confirm that record types can be populated from outputEval."""
+    tool_path = get_data("tests/wf/record_outputeval.cwl")
+    err_code, stdout, stderr = get_main_output([tool_path])
+    assert err_code == 0
+    result = json.loads(stdout)["references"]
+    assert "genome_fa" in result
+    assert result["genome_fa"]["class"] == "File"
+    assert result["genome_fa"]["basename"] == "GRCm38.primary_assembly.genome.fa"
+    assert (
+        result["genome_fa"]["checksum"]
+        == "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    )
+    assert result["genome_fa"]["size"] == 0
+    assert "annotation_gtf" in result
+    assert result["annotation_gtf"]["class"] == "File"
+    assert (
+        result["annotation_gtf"]["basename"]
+        == "gencode.vM21.primary_assembly.annotation.gtf"
+    )
+    assert (
+        result["annotation_gtf"]["checksum"]
+        == "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709"
+    )
+    assert result["annotation_gtf"]["size"] == 0
+
+
+def tests_outputsource_valid_identifier_invalid_source() -> None:
+    """Confirm error for invalid source that was also a valid identifier."""
+    tool_path = get_data("tests/checker_wf/broken-wf4.cwl")
+    err_code, stdout, stderr = get_main_output([tool_path])
+    assert err_code == 1
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "tests/checker_wf/broken-wf4.cwl:12:5: outputSource not found" in stderr
+    assert "tests/checker_wf/broken-wf4.cwl#echo_w" in stderr

tests/wf/paramref_arguments_roundtrip.cwl

@@ -0,0 +1,41 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  a_record:
+    type:
+      - "null"
+      - type: record
+        fields:
+          first: string
+          second: int
+          third: float
+          fourth: long
+          fifth: double
+          sixth: File
+    default:
+      first: y
+      second: 23
+      third: 2.3
+      fourth: 4242424242
+      fifth: 4200000000000000000000000000000000000000000
+      sixth:
+        class: File
+        path: whale.txt
+arguments:
+ - '{'
+ - '"same_record": $(inputs.a_record)'
+ - '}'
+outputs:
+  same_record:
+    type:
+      type: record
+      fields:
+        first: string
+        second: int
+        third: float
+        fourth: long
+        fifth: double
+        sixth: File
+baseCommand: echo
+stdout: cwl.output.json

tests/wf/record_outputeval.cwl

@@ -0,0 +1,36 @@
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+  InlineJavascriptRequirement: {}
+   
+inputs:
+  gtf_version:
+    type: string
+    default: M21
+  organism:
+    type: string
+    default: mouse
+  organism_prefix:
+    type: string
+    default: m
+
+baseCommand:
+  - bash
+  - -c
+arguments:
+  - touch GRC$(inputs.organism_prefix)38.primary_assembly.genome.fa ; touch  gencode.v$(inputs.gtf_version).primary_assembly.annotation.gtf 
+outputs:
+  - id: references
+    type:
+        name: References
+        fields:
+          - name: genome_fa
+            type: File
+          - name: annotation_gtf
+            type: File
+        type: record
+    outputBinding:
+        outputEval: '$({ "genome_fa": { "class": "File", "path": runtime.outdir+"/"+"GRC"
+            + inputs.organism_prefix + "38.primary_assembly.genome.fa" }, "annotation_gtf":
+            { "class": "File", "path": runtime.outdir+"/"+"gencode.v" + inputs.gtf_version
+            + ".primary_assembly.annotation.gtf" } })'