From fbed7db0ce04bd119279cef221822242c8632129 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Thu, 11 Nov 2021 12:13:09 +0900 Subject: [PATCH 1/9] WIP fix #1330 --- cwltool/process.py | 4 ++++ cwltool/workflow.py | 9 +++++---- cwltool/workflow_job.py | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cwltool/process.py b/cwltool/process.py index f33ff0739..3580b96da 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -775,6 +775,10 @@ def _init_job( ) job = copy.deepcopy(joborder) + if len(job.keys()) > 0: + for jo in joborder: + field = shortname(jo) + job[field] = joborder[jo] make_fs_access = getdefault(runtime_context.make_fs_access, StdFsAccess) fs_access = make_fs_access(runtime_context.basedir) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index c96484d95..5ad91f7b9 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -440,10 +440,11 @@ def job( ) step_input = {} - for inp in self.tool["inputs"]: - field = shortname(inp["id"]) - if not inp.get("not_connected"): - step_input[field] = job_order[inp["id"]] + for inp in job_order: + field = shortname(inp) + step_input[field] = job_order[inp] + # if not inp.get("not_connected"): + # step_input[field] = job_order[field] try: yield from self.embedded_tool.job( diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 0bb3472ce..2460e9942 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -386,6 +386,8 @@ def object_from_state( incomplete: bool = False, ) -> Optional[CWLObjectType]: inputobj = {} # type: CWLObjectType + for s in state.keys(): + inputobj[s] = state[s].value for inp in parms: iid = original_id = cast(str, inp["id"]) if frag_only: From 2414df97e054383b03e707ebd413be656f1e2ca0 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Wed, 30 Mar 2022 01:40:35 +0900 Subject: [PATCH 2/9] Revert cwltool/process.py --- cwltool/process.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cwltool/process.py b/cwltool/process.py index 3580b96da..f33ff0739 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -775,10 +775,6 @@ def _init_job( ) job = copy.deepcopy(joborder) - if len(job.keys()) > 0: - for jo in joborder: - field = shortname(jo) - job[field] = joborder[jo] make_fs_access = getdefault(runtime_context.make_fs_access, StdFsAccess) fs_access = make_fs_access(runtime_context.basedir) From 1862895cd0251f69c62ac5501afe8f7e31de7250 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Wed, 30 Mar 2022 01:50:12 +0900 Subject: [PATCH 3/9] Update to fix #1330 --- cwltool/workflow.py | 6 +++--- cwltool/workflow_job.py | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 5ad91f7b9..88dc16d5f 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -443,9 +443,9 @@ def job( for inp in job_order: field = shortname(inp) step_input[field] = job_order[inp] - # if not inp.get("not_connected"): - # step_input[field] = job_order[field] - + if hasattr(inp, "get"): + if not inp.get("not_connected"): + step_input[field] = job_order[field] try: yield from self.embedded_tool.job( step_input, diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 2460e9942..9e39389d7 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -387,7 +387,8 @@ def object_from_state( ) -> Optional[CWLObjectType]: inputobj = {} # type: CWLObjectType for s in state.keys(): - inputobj[s] = state[s].value + if hasattr(state[s], "value"): + inputobj[s] = state[s].value for inp in parms: iid = original_id = cast(str, inp["id"]) if frag_only: @@ -553,6 +554,23 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: _logger.debug("[%s] outputs %s", self.name, json_dumps(wo, indent=4)) self.did_callback = True + paramskey=[] + paramskeys=[] + + if self.tool["outputs"] is not None: + for inp in self.tool["outputs"]: + for k,v in inp.items(): + if k == 'id': + paramskey.append(v) + paramskeys.append(shortname(v)) + + newwo={} + for k,v in wo.items(): + if k in paramskey: + newwo[k]=v + elif k in paramskeys: + newwo[k]=v + wo = newwo final_output_callback(wo, self.processStatus) From 59c97cfe596672574c4e1ee9184812631c6354b1 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Wed, 30 Mar 2022 10:26:47 +0900 Subject: [PATCH 4/9] Fix lint problem --- cwltool/workflow_job.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index b262ccb93..3a43df16a 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -554,22 +554,22 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: _logger.debug("[%s] outputs %s", self.name, json_dumps(wo, indent=4)) self.did_callback = True - paramskey=[] - paramskeys=[] + paramskey = [] + paramskeys = [] if self.tool["outputs"] is not None: for inp in self.tool["outputs"]: - for k,v in inp.items(): - if k == 'id': + for k, v in inp.items(): + if k == "id": paramskey.append(v) paramskeys.append(shortname(v)) - newwo={} - for k,v in wo.items(): + newwo = {} + for k, v in wo.items(): if k in paramskey: - newwo[k]=v + newwo[k] = v elif k in paramskeys: - newwo[k]=v + newwo[k] = v wo = newwo final_output_callback(wo, self.processStatus) From a7a66cf340fd221a3a7e591b39e0647c6229e791 Mon Sep 17 00:00:00 2001 From: Manabu ISHII Date: Thu, 31 Mar 2022 07:46:01 +0900 Subject: [PATCH 5/9] Fix for mypy --- cwltool/workflow.py | 9 +++++---- cwltool/workflow_job.py | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index fe33e8988..245c61e77 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -448,12 +448,13 @@ def job( ) step_input = {} - for inp in job_order: + for inp in dict(job_order).keys(): field = shortname(inp) step_input[field] = job_order[inp] - if hasattr(inp, "get"): - if not inp.get("not_connected"): - step_input[field] = job_order[field] + # if not isinstance(inp, str): + # if hasattr(inp, "get"): + # if not inp.get("not_connected"): + # step_input[field] = job_order[field] try: yield from self.embedded_tool.job( step_input, diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 3a43df16a..77fbaa28c 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -386,9 +386,12 @@ def object_from_state( incomplete: bool = False, ) -> Optional[CWLObjectType]: inputobj = {} # type: CWLObjectType - for s in state.keys(): - if hasattr(state[s], "value"): - inputobj[s] = state[s].value + if state is not None: + for s in state.keys(): + ss = state[s] + if ss is not None: + if isinstance(ss, WorkflowStateItem): + inputobj[s] = ss.value for inp in params: iid = original_id = cast(str, inp["id"]) if frag_only: @@ -564,13 +567,14 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: paramskey.append(v) paramskeys.append(shortname(v)) - newwo = {} - for k, v in wo.items(): - if k in paramskey: - newwo[k] = v - elif k in paramskeys: - newwo[k] = v - wo = newwo + if wo is not None: + newwo = {} + for k, v in dict(wo).items(): + if k in paramskey: + newwo[k] = v + elif k in paramskeys: + newwo[k] = v + wo = newwo final_output_callback(wo, self.processStatus) From 8031b3d02eacfc2ce8aa0adc08056c4b71385ad7 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Thu, 31 Mar 2022 21:52:06 +0900 Subject: [PATCH 6/9] Fix podman v1.0 error --- cwltool/workflow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 245c61e77..1381fe0b6 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -448,6 +448,10 @@ def job( ) step_input = {} + for inp in self.tool["inputs"]: + field = shortname(inp["id"]) + if not inp.get("not_connected"): + step_input[field] = job_order[inp["id"]] for inp in dict(job_order).keys(): field = shortname(inp) step_input[field] = job_order[inp] From a6caace14ea705f9f763938fd5139b8374946935 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Fri, 1 Apr 2022 23:08:50 +0900 Subject: [PATCH 7/9] Remove items for evalResources --- cwltool/command_line_tool.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index c298e1aff..052103804 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -989,6 +989,23 @@ def update_status_output_callback( builder = self._init_job(job_order, runtimeContext) + paramskey = [] + paramskeys = [] + + if self.tool["inputs"] is not None: + for inp in self.tool["inputs"]: + for k, v in inp.items(): + if k == "id": + paramskey.append(v) + paramskeys.append(shortname(v)) + if builder.job is not None: + newbj = {} + for k, v in dict(builder.job).items(): + if k in paramskey: + newbj[k] = v + elif k in paramskeys: + newbj[k] = v + builder.job = newbj reffiles = copy.deepcopy(builder.files) j = self.make_job_runner(runtimeContext)( From 8c0831806757d7c14e6b5b5837e93f4b6a4507d9 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Tue, 5 Apr 2022 22:41:51 +0900 Subject: [PATCH 8/9] Add expressiontool conformance test case --- cwltool/schemas/v1.1/conformance_tests.yaml | 11 ++++++++++ tests/fail-unconnected-expressiontool.cwl | 20 +++++++++++++++++++ .../fail-unspecified-input-expressiontool.cwl | 18 +++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 tests/fail-unconnected-expressiontool.cwl create mode 100644 tests/fail-unspecified-input-expressiontool.cwl diff --git a/cwltool/schemas/v1.1/conformance_tests.yaml b/cwltool/schemas/v1.1/conformance_tests.yaml index a22af165a..ea2595f54 100644 --- a/cwltool/schemas/v1.1/conformance_tests.yaml +++ b/cwltool/schemas/v1.1/conformance_tests.yaml @@ -2598,6 +2598,17 @@ doc: Test an anonymous enum inside an array inside a record, SchemaDefRequirement tags: [command_line_tool, schema_def] + +- job: tests/empty.json + tool: tests/fail-unconnected-expressiontool.cwl + label: wf_step_access_undeclared_param_expressiontool + id: 198 + doc: >- + Test that parameters that don't appear in the `run` process + inputs are not present in the input object used to run the expressiontool. + should_fail: true + tags: [ required, workflow ] + # New tests for v1.1 - job: tests/wc-job.json diff --git a/tests/fail-unconnected-expressiontool.cwl b/tests/fail-unconnected-expressiontool.cwl new file mode 100644 index 000000000..05fbf9060 --- /dev/null +++ b/tests/fail-unconnected-expressiontool.cwl @@ -0,0 +1,20 @@ +class: Workflow +cwlVersion: v1.1 +inputs: + inp1: + type: string + default: hello inp1 + inp2: + type: string + default: hello inp2 +outputs: + out: + type: string + outputSource: step1/out +steps: + step1: + in: + in: inp1 + in2: inp2 + out: [out] + run: fail-unspecified-input-expressiontool.cwl diff --git a/tests/fail-unspecified-input-expressiontool.cwl b/tests/fail-unspecified-input-expressiontool.cwl new file mode 100644 index 000000000..e9cf15aac --- /dev/null +++ b/tests/fail-unspecified-input-expressiontool.cwl @@ -0,0 +1,18 @@ +cwlVersion: v1.0 +class: ExpressionTool + +requirements: + InlineJavascriptRequirement: {} + +inputs: + in: + type: string + +outputs: + out: + type: string + +expression: | + ${ + return {"out": inputs.in +" "+inputs.in2}; + } From 14a2dedf63b240c4bb3f455292e7fda861b0ee03 Mon Sep 17 00:00:00 2001 From: Manabu Ishii Date: Thu, 30 Jun 2022 17:27:56 +0900 Subject: [PATCH 9/9] Improve ExpressionTool inputs It is not passed parameters which is not pass to ExpressionTool --- cwltool/command_line_tool.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 052103804..b21dcdc23 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -201,6 +201,24 @@ def job( ) -> Generator[ExpressionJob, None, None]: builder = self._init_job(job_order, runtimeContext) + paramskey = [] + paramskeys = [] + + if self.tool["inputs"] is not None: + for inp in self.tool["inputs"]: + for k, v in inp.items(): + if k == "id": + paramskey.append(v) + paramskeys.append(shortname(v)) + if builder.job is not None: + newbj = {} + for k, v in dict(builder.job).items(): + if k in paramskey: + newbj[k] = v + elif k in paramskeys: + newbj[k] = v + builder.job = newbj + job = ExpressionJob( builder, self.tool["expression"],