diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 0d8f24697..fe0afbbaa 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -656,8 +656,10 @@ def arg_parser() -> argparse.ArgumentParser: default=None, help="Only executes the underlying Process (CommandLineTool, " "ExpressionTool, or sub-Workflow) for the given step in a workflow. " - "This will not include any step-level processing: scatter, when, no " - "processing of step-level default, or valueFrom input modifiers. " + "This will not include any step-level processing: 'scatter', 'when'; " + "and there will be no processing of step-level 'default', or 'valueFrom' " + "input modifiers. However, requirements/hints from the step or parent " + "workflow(s) will be inherited as usual." "The input object must match that Process's inputs.", ) diff --git a/cwltool/builder.py b/cwltool/builder.py index 191ef9ee7..71f7365a6 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -11,7 +11,6 @@ MutableSequence, Optional, Set, - Tuple, Union, cast, ) @@ -36,6 +35,7 @@ CONTENT_LIMIT, CWLObjectType, CWLOutputType, + HasReqsHints, aslist, get_listing, normalizeFilesDirs, @@ -134,26 +134,6 @@ def check_format( ) -class HasReqsHints: - """Base class for get_requirement().""" - - def __init__(self) -> None: - """Initialize this reqs decorator.""" - self.requirements = [] # type: List[CWLObjectType] - self.hints = [] # type: List[CWLObjectType] - - def get_requirement( - self, feature: str - ) -> Tuple[Optional[CWLObjectType], Optional[bool]]: - for item in reversed(self.requirements): - if item["class"] == feature: - return (item, True) - for item in reversed(self.hints): - if item["class"] == feature: - return (item, False) - return (None, None) - - class Builder(HasReqsHints): def __init__( self, diff --git a/cwltool/context.py b/cwltool/context.py index 9661b149e..2fc1f0e2e 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -12,14 +12,14 @@ from schema_salad.utils import FetcherCallableType from typing_extensions import TYPE_CHECKING -from .builder import Builder, HasReqsHints +from .builder import Builder from .mpi import MpiConfig from .mutation import MutationManager from .pathmapper import PathMapper from .secrets import SecretStore from .software_requirements import DependenciesConfiguration from .stdfsaccess import StdFsAccess -from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, ResolverType +from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: from .process import Process diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e69668f37..6a5f640d8 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,5 +1,4 @@ """Visualize a CWL workflow.""" -import os from pathlib import Path from urllib.parse import urlparse diff --git a/cwltool/job.py b/cwltool/job.py index 0e7b09a2f..e617fcd96 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -39,7 +39,7 @@ from typing_extensions import TYPE_CHECKING from . import env_to_stdout, run_job -from .builder import Builder, HasReqsHints +from .builder import Builder from .context import RuntimeContext from .errors import UnsupportedRequirement, WorkflowException from .loghandler import _logger @@ -49,6 +49,7 @@ from .utils import ( CWLObjectType, CWLOutputType, + HasReqsHints, DirectoryType, OutputCallbackType, bytes2str_in_dicts, diff --git a/cwltool/main.py b/cwltool/main.py index bad346db6..6d4f9e3b8 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -40,12 +40,11 @@ from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri, uri_file_path -from schema_salad.sourceline import strip_dup_lineno +from schema_salad.sourceline import cmap, strip_dup_lineno from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts from . import CWL_CONTENT_TYPES, workflow from .argparser import arg_parser, generate_parser, get_default_args -from .builder import HasReqsHints from .context import LoadingContext, RuntimeContext, getdefault from .cwlrdf import printdot, printrdf from .errors import ArgumentException, UnsupportedRequirement, WorkflowException @@ -89,6 +88,7 @@ CWLObjectType, CWLOutputAtomType, CWLOutputType, + HasReqsHints, adjustDirObjs, normalizeFilesDirs, processes_to_kill, @@ -754,37 +754,63 @@ def my_represent_none( ) +def inherit_reqshints(tool: Process, parent: Process) -> None: + """Copy down requirements and hints from ancestors of a given process.""" + for parent_req in parent.requirements: + found = False + for tool_req in tool.requirements: + if parent_req["class"] == tool_req["class"]: + found = True + break + if not found: + tool.requirements.append(parent_req) + for parent_hint in parent.hints: + found = False + for tool_req in tool.requirements: + if parent_hint["class"] == tool_req["class"]: + found = True + break + if not found: + for tool_hint in tool.hints: + if parent_hint["class"] == tool_hint["class"]: + found = True + break + if not found: + tool.hints.append(parent_hint) + + def choose_target( args: argparse.Namespace, tool: Process, - loadingContext: LoadingContext, + loading_context: LoadingContext, ) -> Optional[Process]: """Walk the Workflow, extract the subset matches all the args.targets.""" - if loadingContext.loader is None: - raise Exception("loadingContext.loader cannot be None") + if loading_context.loader is None: + raise Exception("loading_context.loader cannot be None") if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: extracted = get_subgraph( - [tool.tool["id"] + "/" + r for r in args.target], tool + [tool.tool["id"] + "/" + r for r in args.target], tool, loading_context ) else: extracted = get_subgraph( [ - loadingContext.loader.fetcher.urljoin(tool.tool["id"], "#" + r) + loading_context.loader.fetcher.urljoin(tool.tool["id"], "#" + r) for r in args.target ], tool, + loading_context, ) else: _logger.error("Can only use --target on Workflows") return None - if isinstance(loadingContext.loader.idx, MutableMapping): - loadingContext.loader.idx[extracted["id"]] = extracted - tool = make_tool(extracted["id"], loadingContext) + if isinstance(loading_context.loader.idx, MutableMapping): + loading_context.loader.idx[extracted["id"]] = extracted + tool = make_tool(extracted["id"], loading_context) else: - raise Exception("Missing loadingContext.loader.idx!") + raise Exception("Missing loading_context.loader.idx!") return tool @@ -792,31 +818,31 @@ def choose_target( def choose_step( args: argparse.Namespace, tool: Process, - loadingContext: LoadingContext, + loading_context: LoadingContext, ) -> Optional[Process]: """Walk the given Workflow and extract just args.single_step.""" - if loadingContext.loader is None: - raise Exception("loadingContext.loader cannot be None") + if loading_context.loader is None: + raise Exception("loading_context.loader cannot be None") if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: - extracted = get_step(tool, tool.tool["id"] + "/" + args.single_step) + step_id = tool.tool["id"] + "/" + args.single_step else: - extracted = get_step( - tool, - loadingContext.loader.fetcher.urljoin( - tool.tool["id"], "#" + args.single_step - ), + step_id = loading_context.loader.fetcher.urljoin( + tool.tool["id"], "#" + args.single_step ) + extracted = get_step(tool, step_id, loading_context) else: _logger.error("Can only use --single-step on Workflows") return None - if isinstance(loadingContext.loader.idx, MutableMapping): - loadingContext.loader.idx[extracted["id"]] = extracted - tool = make_tool(extracted["id"], loadingContext) + if isinstance(loading_context.loader.idx, MutableMapping): + loading_context.loader.idx[extracted["id"]] = cast( + Union[CommentedMap, CommentedSeq, str, None], cmap(extracted) + ) + tool = make_tool(extracted["id"], loading_context) else: - raise Exception("Missing loadingContext.loader.idx!") + raise Exception("Missing loading_context.loader.idx!") return tool @@ -826,36 +852,33 @@ def choose_process( tool: Process, loadingContext: LoadingContext, ) -> Optional[Process]: - """Walk the given Workflow and extract just args.single_step.""" + """Walk the given Workflow and extract just args.single_process.""" if loadingContext.loader is None: raise Exception("loadingContext.loader cannot be None") if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: - extracted = get_process( - tool, - tool.tool["id"] + "/" + args.single_process, - loadingContext.loader.idx, - ) + step_id = tool.tool["id"] + "/" + args.single_process else: - extracted = get_process( - tool, - loadingContext.loader.fetcher.urljoin( - tool.tool["id"], "#" + args.single_process - ), - loadingContext.loader.idx, + step_id = loadingContext.loader.fetcher.urljoin( + tool.tool["id"], "#" + args.single_process ) + extracted, workflow_step = get_process( + tool, + step_id, + loadingContext, + ) else: _logger.error("Can only use --single-process on Workflows") return None if isinstance(loadingContext.loader.idx, MutableMapping): loadingContext.loader.idx[extracted["id"]] = extracted - tool = make_tool(extracted["id"], loadingContext) + new_tool = make_tool(extracted["id"], loadingContext) else: raise Exception("Missing loadingContext.loader.idx!") - - return tool + inherit_reqshints(new_tool, workflow_step) + return new_tool def check_working_directories( @@ -887,6 +910,33 @@ def check_working_directories( return None +def print_targets( + tool: Process, + stdout: Union[TextIO, StreamWriter], + loading_context: LoadingContext, + prefix: str = "", +) -> None: + """Recursively find targets for --subgraph and friends.""" + for f in ("outputs", "inputs"): + if tool.tool[f]: + _logger.info("%s %s%s targets:", prefix[:-1], f[0].upper(), f[1:-1]) + stdout.write( + " " + + "\n ".join([f"{prefix}{shortname(t['id'])}" for t in tool.tool[f]]) + + "\n" + ) + if "steps" in tool.tool: + _logger.info("%s steps targets:", prefix[:-1]) + for t in tool.tool["steps"]: + stdout.write(f" {prefix}{shortname(t['id'])}\n") + run: Union[str, Process] = t["run"] + if isinstance(run, str): + process = make_tool(run, loading_context) + else: + process = run + print_targets(process, stdout, loading_context, shortname(t["id"]) + "/") + + def main( argsl: Optional[List[str]] = None, args: Optional[argparse.Namespace] = None, @@ -1084,14 +1134,7 @@ def main( return 0 if args.print_targets: - for f in ("outputs", "steps", "inputs"): - if tool.tool[f]: - _logger.info("%s%s targets:", f[0].upper(), f[1:-1]) - stdout.write( - " " - + "\n ".join([shortname(t["id"]) for t in tool.tool[f]]) - + "\n" - ) + print_targets(tool, stdout, loadingContext) return 0 if args.target: diff --git a/cwltool/process.py b/cwltool/process.py index ddb6a9040..4608f979e 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -49,7 +49,7 @@ from typing_extensions import TYPE_CHECKING from . import expression -from .builder import INPUT_OBJ_VOCAB, Builder, HasReqsHints +from .builder import INPUT_OBJ_VOCAB, Builder from .context import LoadingContext, RuntimeContext, getdefault from .errors import UnsupportedRequirement, WorkflowException from .loghandler import _logger @@ -62,6 +62,7 @@ CWLObjectType, CWLOutputAtomType, CWLOutputType, + HasReqsHints, JobsGeneratorType, OutputCallbackType, adjustDirObjs, diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 1928a763c..b91946160 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -14,8 +14,10 @@ from typing_extensions import TYPE_CHECKING +from .utils import HasReqsHints + if TYPE_CHECKING: - from .builder import Builder, HasReqsHints + from .builder import Builder try: from galaxy.tool_util import deps @@ -98,7 +100,7 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: return job_script -def get_dependencies(builder: "HasReqsHints") -> ToolRequirements: +def get_dependencies(builder: HasReqsHints) -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") dependencies = [] # type: List[ToolRequirement] if software_requirement and software_requirement.get("packages"): @@ -129,7 +131,7 @@ def get_dependencies(builder: "HasReqsHints") -> ToolRequirements: def get_container_from_software_requirements( - use_biocontainers: bool, builder: "HasReqsHints" + use_biocontainers: bool, builder: HasReqsHints ) -> Optional[str]: if use_biocontainers: ensure_galaxy_lib_available() diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index ec4537fc2..c985c59fd 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -10,11 +10,15 @@ Optional, Set, Tuple, + Union, cast, ) -from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.comments import CommentedMap, CommentedSeq +from .context import LoadingContext +from .load_tool import load_tool +from .process import Process from .utils import CWLObjectType, aslist from .workflow import Workflow, WorkflowStep @@ -55,14 +59,41 @@ def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node return nodes[nodeid] -def find_step(steps: List[WorkflowStep], stepid: str) -> Optional[CWLObjectType]: +def find_step( + steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: if st.tool["id"] == stepid: - return st.tool - return None - - -def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: + return st.tool, st + if stepid.startswith(st.tool["id"]): + run: Union[str, Process] = st.tool["run"] + if isinstance(run, Workflow): + result, st2 = find_step( + run.steps, stepid[len(st.tool["id"]) + 1 :], loading_context + ) + if result: + return result, st2 + elif isinstance(run, str): + process = load_tool(run, loading_context) + if isinstance(process, Workflow): + suffix = stepid[len(st.tool["id"]) + 1 :] + prefix = process.tool["id"] + if "#" in prefix: + sep = "/" + else: + sep = "#" + adj_stepid = f"{prefix}{sep}{suffix}" + result2, st3 = find_step(process.steps, adj_stepid, loading_context) + if result2: + return result2, st3 + return None, None + + +def get_subgraph( + roots: MutableSequence[str], tool: Workflow, loading_context: LoadingContext +) -> CommentedMap: + """Extract the subgraph for the given roots.""" if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") @@ -73,7 +104,7 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: for out in tool.tool["outputs"]: declare_node(nodes, out["id"], OUTPUT) - for i in aslist(out.get("outputSource", [])): + for i in aslist(out.get("outputSource", CommentedSeq)): # source is upstream from output (dependency) nodes[out["id"]].up.append(i) # output is downstream from source @@ -124,7 +155,7 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: df = urllib.parse.urldefrag(u) rn = str(df[0] + "#" + df[1].replace("/", "_")) if nodes[v].type == STEP: - wfstep = find_step(tool.steps, v) + wfstep = find_step(tool.steps, v, loading_context)[0] if wfstep is not None: for inp in cast( MutableSequence[CWLObjectType], wfstep["inputs"] @@ -140,7 +171,7 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: extracted = CommentedMap() for f in tool.tool: if f in ("steps", "inputs", "outputs"): - extracted[f] = [] + extracted[f] = CommentedSeq() for i in tool.tool[f]: if i["id"] in visited: if f == "steps": @@ -148,11 +179,13 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: if "source" not in inport: continue if isinstance(inport["source"], MutableSequence): - inport["source"] = [ - rewire[s][0] - for s in inport["source"] - if s in rewire - ] + inport["source"] = CommentedSeq( + [ + rewire[s][0] + for s in inport["source"] + if s in rewire + ] + ) elif inport["source"] in rewire: inport["source"] = rewire[inport["source"]][0] extracted[f].append(i) @@ -160,26 +193,30 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: extracted[f] = tool.tool[f] for rv in rewire.values(): - extracted["inputs"].append({"id": rv[0], "type": rv[1]}) + extracted["inputs"].append(CommentedMap({"id": rv[0], "type": rv[1]})) return extracted -def get_step(tool: Workflow, step_id: str) -> CommentedMap: - +def get_step( + tool: Workflow, step_id: str, loading_context: LoadingContext +) -> CommentedMap: + """Extract a single WorkflowStep for the given step_id.""" extracted = CommentedMap() - step = find_step(tool.steps, step_id) + step = find_step(tool.steps, step_id, loading_context)[0] if step is None: raise Exception(f"Step {step_id} was not found") - extracted["steps"] = [step] - extracted["inputs"] = [] - extracted["outputs"] = [] + new_id, step_name = cast(str, step["id"]).rsplit("#") + + extracted["steps"] = CommentedSeq([step]) + extracted["inputs"] = CommentedSeq() + extracted["outputs"] = CommentedSeq() for inport in cast(List[CWLObjectType], step["in"]): - name = cast(str, inport["id"]).split("#")[-1].split("/")[-1] - extracted["inputs"].append({"id": name, "type": "Any"}) + name = "#" + cast(str, inport["id"]).split("#")[-1].split("/")[-1] + extracted["inputs"].append(CommentedMap({"id": name, "type": "Any"})) inport["source"] = name if "linkMerge" in inport: del inport["linkMerge"] @@ -187,25 +224,36 @@ def get_step(tool: Workflow, step_id: str) -> CommentedMap: for outport in cast(List[str], step["out"]): name = outport.split("#")[-1].split("/")[-1] extracted["outputs"].append( - {"id": name, "type": "Any", "outputSource": f"{step_id}/{name}"} + { + "id": name, + "type": "Any", + "outputSource": f"{new_id}#{step_name}/{name}", + } ) for f in tool.tool: if f not in ("steps", "inputs", "outputs"): extracted[f] = tool.tool[f] - + extracted["id"] = new_id + if "cwlVersion" not in extracted: + extracted["cwlVersion"] = tool.metadata["cwlVersion"] return extracted -def get_process(tool: Workflow, step_id: str, index: Mapping[str, Any]) -> Any: - """Return just a single Process from a Workflow step.""" - step = find_step(tool.steps, step_id) - if step is None: +def get_process( + tool: Workflow, step_id: str, loading_context: LoadingContext +) -> Tuple[Any, WorkflowStep]: + """Find the underlying Process for a given Workflow step id.""" + if loading_context.loader is None: + raise Exception("loading_context.loader cannot be None") + raw_step, step = find_step(tool.steps, step_id, loading_context) + if raw_step is None or step is None: raise Exception(f"Step {step_id} was not found") - run = step["run"] + run: Union[str, Any] = raw_step["run"] if isinstance(run, str): - return index[run] + process = loading_context.loader.idx[run] else: - return run + process = run + return process, step diff --git a/cwltool/utils.py b/cwltool/utils.py index aca19536e..2f7020e51 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -29,6 +29,7 @@ NamedTuple, Optional, Set, + Tuple, Union, cast, ) @@ -485,3 +486,23 @@ def create_tmp_dir(tmpdir_prefix: str) -> str: """Create a temporary directory that respects the given tmpdir_prefix.""" tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix) return tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir) + + +class HasReqsHints: + """Base class for get_requirement().""" + + def __init__(self) -> None: + """Initialize this reqs decorator.""" + self.requirements: List[CWLObjectType] = [] + self.hints: List[CWLObjectType] = [] + + def get_requirement( + self, feature: str + ) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + for item in reversed(self.requirements): + if item["class"] == feature: + return (item, True) + for item in reversed(self.hints): + if item["class"] == feature: + return (item, False) + return (None, None) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index e18f31255..c96484d95 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -20,7 +20,7 @@ from schema_salad.sourceline import SourceLine, indent from . import command_line_tool, context, procgenerator -from .checker import static_checker, circular_dependency_checker +from .checker import circular_dependency_checker, static_checker from .context import LoadingContext, RuntimeContext, getdefault from .errors import WorkflowException from .load_tool import load_tool diff --git a/tests/subgraph/env-job.json b/tests/subgraph/env-job.json new file mode 100644 index 000000000..dabf28850 --- /dev/null +++ b/tests/subgraph/env-job.json @@ -0,0 +1,3 @@ +{ + "in": "hello test env" +} diff --git a/tests/subgraph/env-tool2.cwl b/tests/subgraph/env-tool2.cwl new file mode 100644 index 000000000..f568e5450 --- /dev/null +++ b/tests/subgraph/env-tool2.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + in: string +outputs: + out: + type: File + outputBinding: + glob: out + +hints: + EnvVarRequirement: + envDef: + TEST_ENV: $(inputs.in) + +baseCommand: ["/bin/sh", "-c", "echo $TEST_ENV"] + +stdout: out diff --git a/tests/subgraph/env-tool2_no_env.cwl b/tests/subgraph/env-tool2_no_env.cwl new file mode 100644 index 000000000..c4a6f6327 --- /dev/null +++ b/tests/subgraph/env-tool2_no_env.cwl @@ -0,0 +1,13 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + in: string +outputs: + out: + type: File + outputBinding: + glob: out + +baseCommand: ["/bin/sh", "-c", "echo $TEST_ENV"] + +stdout: out diff --git a/tests/subgraph/env-tool2_req.cwl b/tests/subgraph/env-tool2_req.cwl new file mode 100644 index 000000000..96b0bc3d1 --- /dev/null +++ b/tests/subgraph/env-tool2_req.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: v1.2 +inputs: + in: string +outputs: + out: + type: File + outputBinding: + glob: out + +requirements: + EnvVarRequirement: + envDef: + TEST_ENV: $(inputs.in) + +baseCommand: ["/bin/sh", "-c", "echo $TEST_ENV"] + +stdout: out diff --git a/tests/subgraph/env-wf2.cwl b/tests/subgraph/env-wf2.cwl new file mode 100644 index 000000000..b9408fc4d --- /dev/null +++ b/tests/subgraph/env-wf2.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +requirements: + EnvVarRequirement: + envDef: + TEST_ENV: override + +steps: + step1: + run: env-tool2.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/env-wf2_hint_collision.cwl b/tests/subgraph/env-wf2_hint_collision.cwl new file mode 100644 index 000000000..ae1f93d28 --- /dev/null +++ b/tests/subgraph/env-wf2_hint_collision.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +hints: + EnvVarRequirement: + envDef: + TEST_ENV: override + +steps: + step1: + run: env-tool2.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/env-wf2_hint_req_collision.cwl b/tests/subgraph/env-wf2_hint_req_collision.cwl new file mode 100644 index 000000000..c414c3291 --- /dev/null +++ b/tests/subgraph/env-wf2_hint_req_collision.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +hints: + EnvVarRequirement: + envDef: + TEST_ENV: override + +steps: + step1: + run: env-tool2_req.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/env-wf2_only_hint.cwl b/tests/subgraph/env-wf2_only_hint.cwl new file mode 100644 index 000000000..4168e4469 --- /dev/null +++ b/tests/subgraph/env-wf2_only_hint.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +hints: + EnvVarRequirement: + envDef: + TEST_ENV: override-from-parent-hint + +steps: + step1: + run: env-tool2_no_env.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/env-wf2_req_collision.cwl b/tests/subgraph/env-wf2_req_collision.cwl new file mode 100644 index 000000000..626a11fd5 --- /dev/null +++ b/tests/subgraph/env-wf2_req_collision.cwl @@ -0,0 +1,23 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: step1/out + +requirements: + EnvVarRequirement: + envDef: + TEST_ENV: override + +steps: + step1: + run: env-tool2_req.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/env-wf2_subwf-packed.cwl b/tests/subgraph/env-wf2_subwf-packed.cwl new file mode 100644 index 000000000..ed119f0f2 --- /dev/null +++ b/tests/subgraph/env-wf2_subwf-packed.cwl @@ -0,0 +1,130 @@ +{ + "$graph": [ + { + "class": "CommandLineTool", + "inputs": [ + { + "type": "string", + "id": "#env-tool2.cwl/in" + } + ], + "hints": [ + { + "envDef": [ + { + "envValue": "$(inputs.in)", + "envName": "TEST_ENV" + } + ], + "class": "EnvVarRequirement" + } + ], + "baseCommand": [ + "/bin/sh", + "-c", + "echo $TEST_ENV" + ], + "stdout": "out", + "id": "#env-tool2.cwl", + "outputs": [ + { + "type": "File", + "outputBinding": { + "glob": "out" + }, + "id": "#env-tool2.cwl/out" + } + ] + }, + { + "class": "Workflow", + "inputs": [ + { + "type": "string", + "id": "#env-wf2.cwl/in" + } + ], + "outputs": [ + { + "type": "File", + "outputSource": "#env-wf2.cwl/step1/out", + "id": "#env-wf2.cwl/out" + } + ], + "requirements": [ + { + "envDef": [ + { + "envValue": "override", + "envName": "TEST_ENV" + } + ], + "class": "EnvVarRequirement" + } + ], + "steps": [ + { + "run": "#env-tool2.cwl", + "in": [ + { + "source": "#env-wf2.cwl/in", + "id": "#env-wf2.cwl/step1/in" + } + ], + "out": [ + "#env-wf2.cwl/step1/out" + ], + "id": "#env-wf2.cwl/step1" + } + ], + "id": "#env-wf2.cwl" + }, + { + "class": "Workflow", + "inputs": [ + { + "type": "string", + "id": "#main/in" + } + ], + "outputs": [ + { + "type": "File", + "outputSource": "#main/sub_wf/out", + "id": "#main/out" + } + ], + "requirements": [ + { + "envDef": [ + { + "envValue": "override_super", + "envName": "TEST_ENV" + } + ], + "class": "EnvVarRequirement" + }, + { + "class": "SubworkflowFeatureRequirement" + } + ], + "steps": [ + { + "run": "#env-wf2.cwl", + "in": [ + { + "source": "#main/in", + "id": "#main/sub_wf/in" + } + ], + "out": [ + "#main/sub_wf/out" + ], + "id": "#main/sub_wf" + } + ], + "id": "#main" + } + ], + "cwlVersion": "v1.2" +} \ No newline at end of file diff --git a/tests/subgraph/env-wf2_subwf.cwl b/tests/subgraph/env-wf2_subwf.cwl new file mode 100644 index 000000000..03fdf2265 --- /dev/null +++ b/tests/subgraph/env-wf2_subwf.cwl @@ -0,0 +1,24 @@ +#!/usr/bin/env cwl-runner +class: Workflow +cwlVersion: v1.2 + +inputs: + in: string + +outputs: + out: + type: File + outputSource: sub_wf/out + +requirements: + SubworkflowFeatureRequirement: {} + EnvVarRequirement: + envDef: + TEST_ENV: override_super + +steps: + sub_wf: + run: env-wf2.cwl + in: + in: in + out: [out] diff --git a/tests/subgraph/single_step1.json b/tests/subgraph/single_step1.json index 8f140adda..6e6b3d5bb 100644 --- a/tests/subgraph/single_step1.json +++ b/tests/subgraph/single_step1.json @@ -1,13 +1,13 @@ {"class": "Workflow", "cwlVersion": "v1.0", "id": "count-lines1-wf.cwl", - "inputs": [{"id": "file1", "type": "Any"}], + "inputs": [{"id": "#file1", "type": "Any"}], "outputs": [{"id": "output", "outputSource": "count-lines1-wf.cwl#step1/output", "type": "Any"}], "steps": [{"id": "count-lines1-wf.cwl#step1", "in": [{"id": "count-lines1-wf.cwl#step1/file1", - "source": "file1"}], + "source": "#file1"}], "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", "type": "File"}, "id": "count-lines1-wf.cwl#step1/file1", diff --git a/tests/subgraph/single_step2.json b/tests/subgraph/single_step2.json index 68d221688..be8b7c916 100644 --- a/tests/subgraph/single_step2.json +++ b/tests/subgraph/single_step2.json @@ -1,13 +1,13 @@ {"class": "Workflow", "cwlVersion": "v1.0", "id": "count-lines1-wf.cwl", - "inputs": [{"id": "file1", "type": "Any"}], + "inputs": [{"id": "#file1", "type": "Any"}], "outputs": [{"id": "output", "outputSource": "count-lines1-wf.cwl#step2/output", "type": "Any"}], "steps": [{"id": "count-lines1-wf.cwl#step2", "in": [{"id": "count-lines1-wf.cwl#step2/file1", - "source": "file1"}], + "source": "#file1"}], "inputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#file1", "inputBinding": {"loadContents": true}, "type": "File"}, diff --git a/tests/subgraph/single_step3.json b/tests/subgraph/single_step3.json index 5ef062b35..5d189dda7 100644 --- a/tests/subgraph/single_step3.json +++ b/tests/subgraph/single_step3.json @@ -1,13 +1,13 @@ {"class": "Workflow", "cwlVersion": "v1.0", "id": "count-lines1-wf.cwl", - "inputs": [{"id": "file1", "type": "Any"}], + "inputs": [{"id": "#file1", "type": "Any"}], "outputs": [{"id": "output", "outputSource": "count-lines1-wf.cwl#step3/output", "type": "Any"}], "steps": [{"id": "count-lines1-wf.cwl#step3", "in": [{"id": "count-lines1-wf.cwl#step3/file1", - "source": "file1"}], + "source": "#file1"}], "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", "type": "File"}, "id": "count-lines1-wf.cwl#step3/file1", diff --git a/tests/subgraph/single_step4.json b/tests/subgraph/single_step4.json index b8c497943..7803462c6 100644 --- a/tests/subgraph/single_step4.json +++ b/tests/subgraph/single_step4.json @@ -1,13 +1,13 @@ {"class": "Workflow", "cwlVersion": "v1.0", "id": "count-lines1-wf.cwl", - "inputs": [{"id": "file1", "type": "Any"}], + "inputs": [{"id": "#file1", "type": "Any"}], "outputs": [{"id": "output", "outputSource": "count-lines1-wf.cwl#step4/output", "type": "Any"}], "steps": [{"id": "count-lines1-wf.cwl#step4", "in": [{"id": "count-lines1-wf.cwl#step4/file1", - "source": "file1"}], + "source": "#file1"}], "inputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#file1", "inputBinding": {"loadContents": true}, "type": "File"}, diff --git a/tests/subgraph/single_step5.json b/tests/subgraph/single_step5.json index 07c91097a..52e5962c1 100644 --- a/tests/subgraph/single_step5.json +++ b/tests/subgraph/single_step5.json @@ -1,14 +1,14 @@ {"class": "Workflow", "cwlVersion": "v1.0", "id": "count-lines1-wf.cwl", - "inputs": [{"id": "file1", "type": "Any"}, {"id": "file3", "type": "Any"}], + "inputs": [{"id": "#file1", "type": "Any"}, {"id": "#file3", "type": "Any"}], "outputs": [{"id": "output", "outputSource": "count-lines1-wf.cwl#step5/output", "type": "Any"}], "steps": [{"id": "count-lines1-wf.cwl#step5", - "in": [{"id": "count-lines1-wf.cwl#step5/file1", "source": "file1"}, + "in": [{"id": "count-lines1-wf.cwl#step5/file1", "source": "#file1"}, {"id": "count-lines1-wf.cwl#step5/file3", - "source": "file3"}], + "source": "#file3"}], "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", "type": "File"}, "id": "count-lines1-wf.cwl#step5/file1", diff --git a/tests/test_content_type.py b/tests/test_content_type.py index fd2654fc8..9f11880c7 100644 --- a/tests/test_content_type.py +++ b/tests/test_content_type.py @@ -1,6 +1,3 @@ -from typing import Any - -import pydot from pytest import LogCaptureFixture from .util import get_main_output diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index ee4cb837d..5c7fd1069 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -9,7 +9,7 @@ from cwltool.subgraph import get_step, get_subgraph from cwltool.workflow import Workflow, default_make_tool -from .util import get_data +from .util import get_data, get_main_output def clean(val: Any, path: str) -> Any: @@ -26,10 +26,10 @@ def clean(val: Any, path: str) -> Any: def test_get_subgraph() -> None: """Compare known correct subgraphs to generated subgraphs.""" - loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) + loading_context = LoadingContext({"construct_tool_object": default_make_tool}) wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri() - loadingContext.do_update = False - tool = load_tool(wf, loadingContext) + loading_context.do_update = False + tool = load_tool(wf, loading_context) sg = Path(get_data("tests/subgraph")).as_uri() @@ -48,31 +48,31 @@ def test_get_subgraph() -> None: "step5", ): assert isinstance(tool, Workflow) - extracted = get_subgraph([wf + "#" + a], tool) + extracted = get_subgraph([wf + "#" + a], tool, loading_context) with open(get_data("tests/subgraph/extract_" + a + ".json")) as f: assert json.load(f) == clean(convert_to_dict(extracted), sg) def test_get_subgraph_long_out_form() -> None: """Compare subgraphs generatation when 'out' is in the long form.""" - loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) + loading_context = LoadingContext({"construct_tool_object": default_make_tool}) wf = Path(get_data("tests/subgraph/1432.cwl")).as_uri() - loadingContext.do_update = False - tool = load_tool(wf, loadingContext) + loading_context.do_update = False + tool = load_tool(wf, loading_context) sg = Path(get_data("tests/")).as_uri() assert isinstance(tool, Workflow) - extracted = get_subgraph([wf + "#step2"], tool) + extracted = get_subgraph([wf + "#step2"], tool, loading_context) with open(get_data("tests/subgraph/extract_step2_1432.json")) as f: assert json.load(f) == clean(convert_to_dict(extracted), sg) def test_get_step() -> None: - loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) + loading_context = LoadingContext({"construct_tool_object": default_make_tool}) wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri() - loadingContext.do_update = False - tool = load_tool(wf, loadingContext) + loading_context.do_update = False + tool = load_tool(wf, loading_context) assert isinstance(tool, Workflow) sg = Path(get_data("tests/subgraph")).as_uri() @@ -84,6 +84,159 @@ def test_get_step() -> None: "step4", "step5", ): - extracted = get_step(tool, wf + "#" + a) + extracted = get_step(tool, wf + "#" + a, loading_context) with open(get_data("tests/subgraph/single_" + a + ".json")) as f: assert json.load(f) == clean(convert_to_dict(extracted), sg) + + +def test_single_process_inherit_reqshints() -> None: + """Inherit reqs and hints from parent(s) with --single-process.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "step1", + get_data("tests/subgraph/env-wf2.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$cdc1e84968261d6a7575b5305945471f8be199b6" + ) + + +def test_single_process_inherit_hints_collision() -> None: + """Inherit reqs and hints --single-process hints collision.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "step1", + get_data("tests/subgraph/env-wf2_hint_collision.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519" + ) + + +def test_single_process_inherit_reqs_collision() -> None: + """Inherit reqs and hints --single-process reqs collision.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "step1", + get_data("tests/subgraph/env-wf2_req_collision.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519" + ) + + +def test_single_process_inherit_reqs_hints_collision() -> None: + """Inherit reqs and hints --single-process reqs + hints collision.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "step1", + get_data("tests/subgraph/env-wf2_hint_req_collision.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519" + ) + + +def test_single_process_inherit_only_hints() -> None: + """Inherit reqs and hints --single-process only hints.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "step1", + get_data("tests/subgraph/env-wf2_only_hint.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$ab5f2a9add5f54622dde555ac8ae9a3000e5ee0a" + ) + + +def test_single_process_subwf_step() -> None: + """Inherit reqs and hints --single-process on sub-workflow step.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "sub_wf/step1", + get_data("tests/subgraph/env-wf2_subwf.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$cdc1e84968261d6a7575b5305945471f8be199b6" + ) + + +def test_single_process_packed_subwf_step() -> None: + """Inherit reqs and hints --single-process on packed sub-workflow step.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-process", + "sub_wf/step1", + get_data("tests/subgraph/env-wf2_subwf-packed.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$cdc1e84968261d6a7575b5305945471f8be199b6" + ) + + +def test_single_step_subwf_step() -> None: + """Inherit reqs and hints --single-step on sub-workflow step.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-step", + "sub_wf/step1", + get_data("tests/subgraph/env-wf2_subwf.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$7608e5669ba454c61fab01c9b133b52a9a7de68c" + ) + + +def test_single_step_packed_subwf_step() -> None: + """Inherit reqs and hints --single-step on packed sub-workflow step.""" + err_code, stdout, stderr = get_main_output( + [ + "--single-step", + "sub_wf/step1", + get_data("tests/subgraph/env-wf2_subwf-packed.cwl"), + get_data("tests/subgraph/env-job.json"), + ] + ) + assert err_code == 0 + assert ( + json.loads(stdout)["out"]["checksum"] + == "sha1$7608e5669ba454c61fab01c9b133b52a9a7de68c" + )