diff --git a/testdata/echo-tool.cwl b/testdata/echo-tool.cwl new file mode 100644 index 00000000..5a9835b8 --- /dev/null +++ b/testdata/echo-tool.cwl @@ -0,0 +1,17 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.2 +inputs: + in: + type: Any + inputBinding: {} +outputs: + out: + type: string + outputBinding: + glob: out.txt + loadContents: true + outputEval: $(self[0].contents) +baseCommand: echo +stdout: out.txt diff --git a/testdata/empty.json b/testdata/empty.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/testdata/empty.json @@ -0,0 +1 @@ +{} diff --git a/testdata/null-expression-echo-job.json b/testdata/null-expression-echo-job.json new file mode 100644 index 00000000..9047fcfe --- /dev/null +++ b/testdata/null-expression-echo-job.json @@ -0,0 +1,3 @@ +{ + "in": null +} diff --git a/testdata/null-expression1-job.json b/testdata/null-expression1-job.json new file mode 100644 index 00000000..b8ffcd56 --- /dev/null +++ b/testdata/null-expression1-job.json @@ -0,0 +1,3 @@ +{ + "i1": null +} \ No newline at end of file diff --git a/testdata/null-expression2-tool.cwl b/testdata/null-expression2-tool.cwl new file mode 100644 index 00000000..2b09ec04 --- /dev/null +++ b/testdata/null-expression2-tool.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + i1: Any + +outputs: + output: int + +expression: "$({'output': (inputs.i1 == 'the-default' ? 1 : 2)})" \ No newline at end of file diff --git a/tests/jschema_validate.py b/tests/jschema_validate.py new file mode 100755 index 00000000..f855116c --- /dev/null +++ b/tests/jschema_validate.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +from pathlib import Path + +from jsonschema.validators import validate +from ruamel.yaml import YAML + +from cwl_utils.inputs_schema_gen import cwl_to_jsonschema +from cwl_utils.parser import load_document_by_uri + + +def main() -> None: + parser = argparse.ArgumentParser(description="test cwl-inputs-schema-gen.") + parser.add_argument( + "--outdir", + type=str, + default=os.path.abspath("."), + help="Output directory. This is present only for cwltest's usage, and it is ignored.", + ) + parser.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + parser.add_argument("--version", action="store_true", help="Print version and exit") + parser.add_argument( + "workflow", + type=str, + nargs="?", + default=None, + metavar="cwl_document", + help="path or URL to a CWL Workflow, " "CommandLineTool, or ExpressionTool.", + ) + parser.add_argument( + "job_order", + nargs=argparse.REMAINDER, + metavar="inputs_object", + help="path or URL to a YAML or JSON " + "formatted description of the required input values for the given " + "`cwl_document`.", + ) + + args = parser.parse_args(sys.argv[1:]) + + if args.version: + print(f"{sys.argv[1]} 0.0.1") + return + + if len(args.job_order) < 1: + job_order = {} + job_order_name = "empty inputs" + else: + yaml = YAML() + job_order_name = args.job_order[0] + job_order = yaml.load(Path(job_order_name)) + + validate( + job_order, + cwl_to_jsonschema(load_document_by_uri(args.workflow)), + ) + + if not args.quiet: + print( + f"Validation of the JSON schema generated from {args.workflow} " + "using {job_order_name} suceeded." + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_inputs_schema_gen.py b/tests/test_inputs_schema_gen.py index 00091b4c..77f6577d 100644 --- a/tests/test_inputs_schema_gen.py +++ b/tests/test_inputs_schema_gen.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for cwl-inputs-schema-gen.""" +import logging from pathlib import Path import pytest @@ -8,11 +9,13 @@ from ruamel.yaml import YAML from cwl_utils.inputs_schema_gen import cwl_to_jsonschema -from cwl_utils.loghandler import _logger as _cwlutilslogger from cwl_utils.parser import load_document_by_uri from .util import get_path +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger() + TEST_PARAMS = [ # Packed Case ( @@ -51,10 +54,10 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: cwl_obj = load_document_by_uri(tool_path.as_uri()) - _cwlutilslogger.info(f"Generating schema for {tool_path.name}") + logger.info(f"Generating schema for {tool_path.name}") json_schema = cwl_to_jsonschema(cwl_obj) - _cwlutilslogger.info( + logger.info( f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) @@ -65,24 +68,24 @@ def test_cwl_inputs_to_jsonschema(tool_path: Path, inputs_path: Path) -> None: try: validate(input_obj, json_schema) except (ValidationError, SchemaError) as err: - _cwlutilslogger.error( + logger.error( f"Validation failed for {inputs_path.name} " f"against schema generated for input {tool_path.name}" ) raise SchemaError(f"{inputs_path.name} failed schema validation") from err -def test_cwl_inputs_to_jsonschema_fails() -> None: +def test_cwl_inputs_to_jsonschema_single_fail() -> None: """Compare tool schema of param 1 against input schema of param 2.""" tool_path: Path = TEST_PARAMS[0][0] inputs_path: Path = TEST_PARAMS[3][1] cwl_obj = load_document_by_uri(tool_path.as_uri()) - _cwlutilslogger.info(f"Generating schema for {tool_path.name}") + logger.info(f"Generating schema for {tool_path.name}") json_schema = cwl_to_jsonschema(cwl_obj) - _cwlutilslogger.info( + logger.info( f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" ) @@ -93,3 +96,54 @@ def test_cwl_inputs_to_jsonschema_fails() -> None: # We expect this to fail with pytest.raises(ValidationError): validate(input_obj, json_schema) + + +BAD_TEST_PARAMS = [ + # Any without defaults cannot be unspecified + ( + get_path("testdata/null-expression2-tool.cwl"), + get_path("testdata/empty.json"), + "'i1' is a required property", + ), + # null to Any type without a default value. + ( + get_path("testdata/null-expression2-tool.cwl"), + get_path("testdata/null-expression1-job.json"), + "None is not valid under any of the given schemas", + ), + # Any without defaults, unspecified + ( + get_path("testdata/echo-tool.cwl"), + get_path("testdata/null-expression-echo-job.json"), + "None is not valid under any of the given schemas", + ), + # JSON null provided for required input + ( + get_path("testdata/echo-tool.cwl"), + get_path("testdata/null-expression1-job.json"), + "'in' is a required property", + ), +] + + +@pytest.mark.parametrize("tool_path,inputs_path,exception_message", BAD_TEST_PARAMS) +def test_cwl_inputs_to_jsonschema_fails( + tool_path: Path, + inputs_path: Path, + exception_message: str, +) -> None: + cwl_obj = load_document_by_uri(tool_path.as_uri()) + + logger.info(f"Generating schema for {tool_path.name}") + json_schema = cwl_to_jsonschema(cwl_obj) + + logger.info( + f"Testing {inputs_path.name} against schema generated for input {tool_path.name}" + ) + + yaml = YAML() + + input_obj = yaml.load(inputs_path) + + with pytest.raises(ValidationError, match=exception_message): + validate(input_obj, json_schema)