cwl-inputs-schema-gen testing utility

testdata/echo-tool.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+  in:
+    type: Any
+    inputBinding: {}
+outputs:
+  out:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self[0].contents)
+baseCommand: echo
+stdout: out.txt

testdata/empty.json

@@ -0,0 +1 @@
+{}

testdata/null-expression-echo-job.json

@@ -0,0 +1,3 @@
+{
+    "in": null
+}

testdata/null-expression1-job.json

@@ -0,0 +1,3 @@
+{
+    "i1": null
+}

testdata/null-expression2-tool.cwl

@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+
+class: ExpressionTool
+requirements:
+  - class: InlineJavascriptRequirement
+cwlVersion: v1.2
+
+inputs:
+  i1: Any
+
+outputs:
+  output: int
+
+expression: "$({'output': (inputs.i1 == 'the-default' ? 1 : 2)})"

tests/jschema_validate.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from jsonschema.validators import validate
+from ruamel.yaml import YAML
+
+from cwl_utils.inputs_schema_gen import cwl_to_jsonschema
+from cwl_utils.parser import load_document_by_uri
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="test cwl-inputs-schema-gen.")
+    parser.add_argument(
+        "--outdir",
+        type=str,
+        default=os.path.abspath("."),
+        help="Output directory. This is present only for cwltest's usage, and it is ignored.",
+    )
+    parser.add_argument(
+        "--quiet", action="store_true", help="Only print warnings and errors."
+    )
+    parser.add_argument("--version", action="store_true", help="Print version and exit")
+    parser.add_argument(
+        "workflow",
+        type=str,
+        nargs="?",
+        default=None,
+        metavar="cwl_document",
+        help="path or URL to a CWL Workflow, " "CommandLineTool, or ExpressionTool.",
+    )
+    parser.add_argument(
+        "job_order",
+        nargs=argparse.REMAINDER,
+        metavar="inputs_object",
+        help="path or URL to a YAML or JSON "
+        "formatted description of the required input values for the given "
+        "`cwl_document`.",
+    )
+
+    args = parser.parse_args(sys.argv[1:])
+
+    if args.version:
+        print(f"{sys.argv[1]} 0.0.1")
+        return
+
+    if len(args.job_order) < 1:
+        job_order = {}
+        job_order_name = "empty inputs"
+    else:
+        yaml = YAML()
+        job_order_name = args.job_order[0]
+        job_order = yaml.load(Path(job_order_name))
+
+    validate(
+        job_order,
+        cwl_to_jsonschema(load_document_by_uri(args.workflow)),
+    )
+
+    if not args.quiet:
+        print(
+            f"Validation of the JSON schema generated from {args.workflow} "
+            "using {job_order_name} suceeded."
+        )
+
+
+if __name__ == "__main__":
+    main()

tests/test_inputs_schema_gen.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 """Tests for cwl-inputs-schema-gen."""
+import logging
 from pathlib import Path
 
 import pytest
@@ -8,11 +9,13 @@
 from ruamel.yaml import YAML
 
 from cwl_utils.inputs_schema_gen import cwl_to_jsonschema
-from cwl_utils.loghandler import _logger as _cwlutilslogger
 from cwl_utils.parser import load_document_by_uri
 
 from .util import get_path
 
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger()
+
 TEST_PARAMS = [
     # Packed Case
     (
@@ -51,10 +54,10 @@
 def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None:
     cwl_obj = load_document_by_uri(tool_path.as_uri())
 
-    _cwlutilslogger.info(f"Generating schema for {tool_path.name}")
+    logger.info(f"Generating schema for {tool_path.name}")
     json_schema = cwl_to_jsonschema(cwl_obj)
 
-    _cwlutilslogger.info(
+    logger.info(
         f"Testing {inputs_path.name} against schema generated for input {tool_path.name}"
     )
 
@@ -65,24 +68,24 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None:
     try:
         validate(input_obj, json_schema)
     except (ValidationError, SchemaError) as err:
-        _cwlutilslogger.error(
+        logger.error(
             f"Validation failed for {inputs_path.name} "
             f"against schema generated for input {tool_path.name}"
         )
         raise SchemaError(f"{inputs_path.name} failed schema validation") from err
 
 
-def test_cwl_inputs_to_jsonschema_fails() -> None:
+def test_cwl_inputs_to_jsonschema_single_fail() -> None:
     """Compare tool schema of param 1 against input schema of param 2."""
     tool_path: Path = TEST_PARAMS[0][0]
     inputs_path: Path = TEST_PARAMS[3][1]
 
     cwl_obj = load_document_by_uri(tool_path.as_uri())
 
-    _cwlutilslogger.info(f"Generating schema for {tool_path.name}")
+    logger.info(f"Generating schema for {tool_path.name}")
     json_schema = cwl_to_jsonschema(cwl_obj)
 
-    _cwlutilslogger.info(
+    logger.info(
         f"Testing {inputs_path.name} against schema generated for input {tool_path.name}"
     )
 
@@ -93,3 +96,54 @@ def test_cwl_inputs_to_jsonschema_fails() -> None:
     # We expect this to fail
     with pytest.raises(ValidationError):
         validate(input_obj, json_schema)
+
+
+BAD_TEST_PARAMS = [
+    # Any without defaults cannot be unspecified
+    (
+        get_path("testdata/null-expression2-tool.cwl"),
+        get_path("testdata/empty.json"),
+        "'i1' is a required property",
+    ),
+    # null to Any type without a default value.
+    (
+        get_path("testdata/null-expression2-tool.cwl"),
+        get_path("testdata/null-expression1-job.json"),
+        "None is not valid under any of the given schemas",
+    ),
+    # Any without defaults, unspecified
+    (
+        get_path("testdata/echo-tool.cwl"),
+        get_path("testdata/null-expression-echo-job.json"),
+        "None is not valid under any of the given schemas",
+    ),
+    # JSON null provided for required input
+    (
+        get_path("testdata/echo-tool.cwl"),
+        get_path("testdata/null-expression1-job.json"),
+        "'in' is a required property",
+    ),
+]
+
+
+@pytest.mark.parametrize("tool_path,inputs_path,exception_message", BAD_TEST_PARAMS)
+def test_cwl_inputs_to_jsonschema_fails(
+    tool_path: Path,
+    inputs_path: Path,
+    exception_message: str,
+) -> None:
+    cwl_obj = load_document_by_uri(tool_path.as_uri())
+
+    logger.info(f"Generating schema for {tool_path.name}")
+    json_schema = cwl_to_jsonschema(cwl_obj)
+
+    logger.info(
+        f"Testing {inputs_path.name} against schema generated for input {tool_path.name}"
+    )
+
+    yaml = YAML()
+
+    input_obj = yaml.load(inputs_path)
+
+    with pytest.raises(ValidationError, match=exception_message):
+        validate(input_obj, json_schema)