diff --git a/cwltool/builder.py b/cwltool/builder.py index 844c585c6..a417eeedb 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -6,7 +6,7 @@ from typing import Any, Union, AnyStr, Callable from .errors import WorkflowException from .stdfsaccess import StdFsAccess -from .pathmapper import PathMapper +from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs CONTENT_LIMIT = 64 * 1024 @@ -17,18 +17,6 @@ def substitute(value, replace): # type: (str, str) -> str else: return value + replace -def adjustFileObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None - """Apply an update function to each File object in the object `rec`.""" - - if isinstance(rec, dict): - if rec.get("class") == "File": - op(rec) - for d in rec: - adjustFileObjs(rec[d], op) - if isinstance(rec, list): - for d in rec: - adjustFileObjs(d, op) - class Builder(object): def __init__(self): # type: () -> None @@ -114,7 +102,7 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): if schema["type"] == "File": self.files.append(datum) if binding and binding.get("loadContents"): - with self.fs_access.open(datum["path"], "rb") as f: + with self.fs_access.open(datum["location"], "rb") as f: datum["contents"] = f.read(CONTENT_LIMIT) if "secondaryFiles" in schema: @@ -124,11 +112,11 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: secondary_eval = self.do_eval(sf, context=datum) if isinstance(secondary_eval, basestring): - sfpath = {"path": secondary_eval, "class": "File"} + sfpath = {"location": secondary_eval, "class": "File"} else: sfpath = secondary_eval else: - sfpath = {"path": substitute(datum["path"], sf), "class": "File"} + sfpath = {"location": substitute(datum["location"], sf), "class": "File"} if isinstance(sfpath, list): datum["secondaryFiles"].extend(sfpath) else: @@ -140,6 +128,10 @@ def _capture_files(f): adjustFileObjs(datum.get("secondaryFiles", []), _capture_files) + if schema["type"] == "Directory": + self.files.append(datum) + + # Position to front of the sort key if binding: for bi in bindings: diff --git a/cwltool/cwltest.py b/cwltool/cwltest.py index 8b37ce390..a7b7f95ff 100755 --- a/cwltool/cwltest.py +++ b/cwltool/cwltest.py @@ -37,10 +37,30 @@ def compare(a, b): # type: (Any, Any) -> bool # ignore empty collections b = {k: v for k, v in b.iteritems() if not isinstance(v, (list, dict)) or len(v) > 0} + elif a.get("class") == "Directory": + if len(a["listing"]) != len(b["listing"]): + return False + for i in a["listing"]: + found = False + for j in b["listing"]: + try: + if compare(i, j): + found = True + break + except: + pass + if not found: + raise CompareFail(u"%s not in %s" % (json.dumps(i, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) + + a = {k: v for k, v in a.iteritems() + if k not in ("path", "location", "listing")} + b = {k: v for k, v in b.iteritems() + if k not in ("path", "location", "listing")} + if len(a) != len(b): raise CompareFail(u"expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) for c in a: - if a.get("class") != "File" or c != "path": + if a.get("class") != "File" or c not in ("path", "location", "listing"): if c not in b: raise CompareFail(u"%s not in %s" % (c, b)) if not compare(a[c], b[c]): diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py index 39fbaed48..d74199b31 100644 --- a/cwltool/draft2tool.py +++ b/cwltool/draft2tool.py @@ -1,29 +1,34 @@ -import avro.schema +import shutil +from functools import partial import json import copy -from .flatten import flatten -from functools import partial import os -from .pathmapper import PathMapper, DockerPathMapper -from .job import CommandLineJob import glob import logging import hashlib import random -from .process import Process, shortname, uniquename, adjustFileObjs -from .errors import WorkflowException -import schema_salad.validate as validate -from .utils import aslist -from . import expression import re import urlparse import tempfile -from .builder import CONTENT_LIMIT, substitute, Builder -import shellescape import errno + +import avro.schema +import yaml +import schema_salad.validate as validate +import shellescape from typing import Callable, Any, Union, Generator, cast -import hashlib -import shutil + +from .process import Process, shortname, uniquename, getListing +from .errors import WorkflowException +from .utils import aslist +from . import expression +from .builder import CONTENT_LIMIT, substitute, Builder, adjustFileObjs, adjustDirObjs +from .pathmapper import PathMapper +from .job import CommandLineJob + +ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._-]+$") + +from .flatten import flatten _logger = logging.getLogger("cwltool") @@ -67,10 +72,9 @@ def job(self, joborder, output_callback, **kwargs): yield j -def remove_hostfs(f): # type: (Dict[str, Any]) -> None - if "hostfs" in f: - del f["hostfs"] - +def remove_path(f): # type: (Dict[str, Any]) -> None + if "path" in f: + del f["path"] def revmap_file(builder, outdir, f): # type: (Builder,str,Dict[str,Any]) -> Union[Dict[str,Any],None] @@ -80,17 +84,15 @@ def revmap_file(builder, outdir, f): to the external directory. """ - if f.get("hostfs"): - return None + if "location" in f: + return f revmap_f = builder.pathmapper.reversemap(f["path"]) if revmap_f: - f["path"] = revmap_f[1] - f["hostfs"] = True + f["location"] = revmap_f[1] return f elif f["path"].startswith(builder.outdir): - f["path"] = os.path.join(outdir, f["path"][len(builder.outdir)+1:]) - f["hostfs"] = True + f["location"] = os.path.join(outdir, f["path"][len(builder.outdir)+1:]) return f else: raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input file pass through." % (f["path"], builder.outdir)) @@ -118,14 +120,11 @@ def __init__(self, toolpath_object, **kwargs): def makeJobRunner(self): # type: () -> CommandLineJob return CommandLineJob() - def makePathMapper(self, reffiles, **kwargs): - # type: (Set[unicode], **Any) -> PathMapper + def makePathMapper(self, reffiles, stagedir, **kwargs): + # type: (Set[Any], unicode, **Any) -> PathMapper dockerReq, _ = self.get_requirement("DockerRequirement") try: - if dockerReq and kwargs.get("use_container"): - return DockerPathMapper(reffiles, kwargs["basedir"]) - else: - return PathMapper(reffiles, kwargs["basedir"]) + return PathMapper(reffiles, kwargs["basedir"], stagedir) except OSError as e: if e.errno == errno.ENOENT: raise WorkflowException(u"Missing input file %s" % e) @@ -139,9 +138,11 @@ def job(self, joborder, output_callback, **kwargs): cacheargs = kwargs.copy() cacheargs["outdir"] = "/out" cacheargs["tmpdir"] = "/tmp" + cacheargs["stagedir"] = "/stage" cachebuilder = self._init_job(joborder, **cacheargs) - cachebuilder.pathmapper = PathMapper(set((f["path"] for f in cachebuilder.files)), - kwargs["basedir"]) + cachebuilder.pathmapper = PathMapper(cachebuilder.files, + kwargs["basedir"], + cachebuilder.stagedir) cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings)) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") @@ -199,7 +200,7 @@ def rm_pending_output_callback(output_callback, jobcachepending, builder = self._init_job(joborder, **kwargs) - reffiles = set((f[u"path"] for f in builder.files)) + reffiles = copy.deepcopy(builder.files) j = self.makeJobRunner() j.builder = builder @@ -223,10 +224,6 @@ def rm_pending_output_callback(output_callback, jobcachepending, builder.pathmapper = None - if self.tool.get("stdin"): - j.stdin = builder.do_eval(self.tool["stdin"]) - reffiles.add(j.stdin) - if self.tool.get("stderr"): j.stderr = builder.do_eval(self.tool["stderr"]) if os.path.isabs(j.stderr) or ".." in j.stderr: @@ -234,42 +231,57 @@ def rm_pending_output_callback(output_callback, jobcachepending, if self.tool.get("stdout"): j.stdout = builder.do_eval(self.tool["stdout"]) - if os.path.isabs(j.stdout) or ".." in j.stdout: + if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise validate.ValidationException("stdout must be a relative path") - builder.pathmapper = self.makePathMapper(reffiles, **kwargs) + builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **kwargs) builder.requirements = j.requirements # map files to assigned path inside a container. We need to also explicitly # walk over input as implicit reassignment doesn't reach everything in builder.bindings def _check_adjust(f): # type: (Dict[str,Any]) -> Dict[str,Any] - if not f.get("containerfs"): - f["path"] = builder.pathmapper.mapper(f["path"])[1] - f["containerfs"] = True + f["path"] = builder.pathmapper.mapper(f["location"])[1] + f["dirname"], f["basename"] = os.path.split(f["path"]) + if f["class"] == "File": + f["nameroot"], f["nameext"] = os.path.splitext(f["basename"]) + if not ACCEPTLIST_RE.match(f["basename"]): + raise WorkflowException("Invalid filename: '%s' contains illegal characters" % (f["basename"])) return f _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) adjustFileObjs(builder.files, _check_adjust) adjustFileObjs(builder.bindings, _check_adjust) + adjustDirObjs(builder.files, _check_adjust) + adjustDirObjs(builder.bindings, _check_adjust) + + if self.tool.get("stdin"): + j.stdin = builder.do_eval(self.tool["stdin"]) + reffiles.append({"class": "File", "path": j.stdin}) _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) - dockerReq = self.get_requirement("DockerRequirement")[0] + dockerReq, _ = self.get_requirement("DockerRequirement") if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) + j.stagedir = None else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir + j.stagedir = builder.stagedir - createFiles = self.get_requirement("CreateFileRequirement")[0] - j.generatefiles = {} - if createFiles: - for t in createFiles["fileDef"]: - j.generatefiles[builder.do_eval(t["filename"])] = copy.deepcopy(builder.do_eval(t["fileContent"])) + initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0] + j.generatefiles = {"class": "Directory", "listing": []} + if initialWorkdir: + if isinstance(initialWorkdir["listing"], (str, unicode)): + j.generatefiles["listing"] = builder.do_eval(initialWorkdir["listing"]) + else: + for t in initialWorkdir["listing"]: + j.generatefiles["listing"].append({"entryname": builder.do_eval(t["entryname"]), + "entry": copy.deepcopy(builder.do_eval(t["entry"]))}) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] @@ -305,12 +317,11 @@ def collect_output_ports(self, ports, builder, outdir): with builder.fs_access.open(custom_output, "r") as f: ret = json.load(f) _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) - adjustFileObjs(ret, remove_hostfs) adjustFileObjs(ret, cast(Callable[[Any], Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(revmap_file, builder, outdir))) - adjustFileObjs(ret, remove_hostfs) + adjustFileObjs(ret, remove_path) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret @@ -319,9 +330,11 @@ def collect_output_ports(self, ports, builder, outdir): try: ret[fragment] = self.collect_output(port, builder, outdir) except Exception as e: + _logger.debug(u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=e) raise WorkflowException(u"Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e)) if ret: - adjustFileObjs(ret, remove_hostfs) + adjustFileObjs(ret, remove_path) + adjustDirObjs(ret, remove_path) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret if ret is not None else {} except validate.ValidationException as e: @@ -345,60 +358,50 @@ def collect_output(self, schema, builder, outdir): for gb in globpatterns: if gb.startswith(outdir): gb = gb[len(outdir)+1:] + elif gb == ".": + gb = outdir elif gb.startswith("/"): raise WorkflowException("glob patterns must not start with '/'") try: - r.extend([{"path": g, "class": "File", "hostfs": True} + r.extend([{"location": g, + "class": "File" if builder.fs_access.isfile(g) else "Directory"} for g in builder.fs_access.glob(os.path.join(outdir, gb))]) except (OSError, IOError) as e: _logger.warn(str(e)) for files in r: - checksum = hashlib.sha1() - with builder.fs_access.open(files["path"], "rb") as f: - contents = f.read(CONTENT_LIMIT) - if binding.get("loadContents"): - files["contents"] = contents - filesize = 0 - while contents != "": - checksum.update(contents) - filesize += len(contents) - contents = f.read(1024*1024) - files["checksum"] = "sha1$%s" % checksum.hexdigest() - files["size"] = filesize - if "format" in schema: - files["format"] = builder.do_eval(schema["format"], context=files) + if files["class"] == "Directory" and "listing" not in files: + getListing(builder.fs_access, files) + else: + checksum = hashlib.sha1() + with builder.fs_access.open(files["location"], "rb") as f: + contents = f.read(CONTENT_LIMIT) + if binding.get("loadContents"): + files["contents"] = contents + filesize = 0 + while contents != "": + checksum.update(contents) + filesize += len(contents) + contents = f.read(1024*1024) + files["checksum"] = "sha1$%s" % checksum.hexdigest() + files["size"] = filesize + if "format" in schema: + files["format"] = builder.do_eval(schema["format"], context=files) optional = False - singlefile = False + single = False if isinstance(schema["type"], list): if "null" in schema["type"]: optional = True - if "File" in schema["type"]: - singlefile = True - elif schema["type"] == "File": - singlefile = True + if "File" in schema["type"] or "Directory" in schema["type"]: + single = True + elif schema["type"] == "File" or schema["type"] == "Directory": + single = True if "outputEval" in binding: - eout = builder.do_eval(binding["outputEval"], context=r) - if singlefile: - # Handle single file outputs not wrapped in a list - if eout is not None and not isinstance(eout, (list, tuple)): - r = [eout] - elif optional and eout is None: - pass - elif (eout is None or len(eout) != 1 or - not isinstance(eout[0], dict) - or "path" not in eout[0]): - raise WorkflowException( - u"Expression must return a file object for %s." - % schema["id"]) - else: - r = [eout] - else: - r = eout + r = builder.do_eval(binding["outputEval"], context=r) - if singlefile: + if single: if not r and not optional: raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns)) elif not r and optional: @@ -420,14 +423,14 @@ def collect_output(self, schema, builder, outdir): primary["secondaryFiles"] = [] for sf in aslist(schema["secondaryFiles"]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: - sfpath = builder.do_eval(sf, context=r) + sfpath = builder.do_eval(sf, context=primary) if isinstance(sfpath, basestring): - sfpath = revmap({"path": sfpath, "class": "File"}) + sfpath = revmap({"location": sfpath, "class": "File"}) else: - sfpath = {"path": substitute(primary["path"], sf), "class": "File", "hostfs": True} + sfpath = {"location": substitute(primary["location"], sf), "class": "File"} for sfitem in aslist(sfpath): - if builder.fs_access.exists(sfitem["path"]): + if builder.fs_access.exists(sfitem["location"]): primary["secondaryFiles"].append(sfitem) if not r and optional: diff --git a/cwltool/job.py b/cwltool/job.py index 01b3d7c14..539c2f37e 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -8,7 +8,7 @@ import sys import requests from . import docker -from .process import get_feature, empty_subtree +from .process import get_feature, empty_subtree, stageFiles from .errors import WorkflowException import shutil import stat @@ -61,7 +61,7 @@ def __init__(self): # type: () -> None self.generatefiles = None # type: Dict[str,Union[Dict[str,str],str]] def run(self, dry_run=False, pull_image=True, rm_container=True, - rm_tmpdir=True, move_outputs=True, **kwargs): + rm_tmpdir=True, move_outputs="move", **kwargs): # type: (bool, bool, bool, bool, bool, **Any) -> Union[Tuple[str,Dict[None,None]],None] if not os.path.exists(self.outdir): os.makedirs(self.outdir) @@ -79,8 +79,9 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") for f in self.pathmapper.files(): - if not os.path.isfile(self.pathmapper.mapper(f)[0]): - raise WorkflowException(u"Required input file %s not found or is not a regular file." % self.pathmapper.mapper(f)[0]) + p = self.pathmapper.mapper(f) + if p.type == "File" and not os.path.isfile(p[0]): + raise WorkflowException(u"Input file %s (at %s) not found or is not a regular file." % (f, self.pathmapper.mapper(f)[0])) img_id = None if docker_req and kwargs.get("use_container") is not False: @@ -94,7 +95,8 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, runtime = ["docker", "run", "-i"] for src in self.pathmapper.files(): vol = self.pathmapper.mapper(src) - runtime.append(u"--volume=%s:%s:ro" % vol) + if vol.type == "File": + runtime.append(u"--volume=%s:%s:ro" % (vol.resolved, vol.target)) runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl")) runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp")) runtime.append(u"--workdir=%s" % ("/var/spool/cwl")) @@ -136,6 +138,8 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, if key in vars_to_preserve and key not in env: env[key] = value + stageFiles(self.pathmapper, os.symlink) + stdin = None # type: Union[IO[Any],int] stderr = None # type: IO[Any] stdout = None # type: IO[Any] @@ -151,7 +155,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, self.name, self.outdir, " \\\n ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]), - u' < %s' % (self.stdin) if self.stdin else '', + u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') @@ -161,22 +165,22 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, outputs = {} # type: Dict[str,str] try: - for t in self.generatefiles: - entry = self.generatefiles[t] - if isinstance(entry, dict): - src = entry["path"] - dst = os.path.join(self.outdir, t) - if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir: - _logger.debug(u"symlinking %s to %s", dst, src) - os.symlink(src, dst) - elif isinstance(entry, (str, unicode)): - with open(os.path.join(self.outdir, t), "w") as fout: - fout.write(entry.encode("utf-8")) - else: - raise Exception("Unhandled type %s", type(entry)) + if self.generatefiles["listing"]: + generatemapper = PathMapper([self.generatefiles], self.outdir, + self.outdir, separateDirs=False) + _logger.debug(u"[job %s] initial work dir %s", self.name, + json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4)) + def linkoutdir(src, tgt): + # Need to make the link to the staged file (may be inside + # the container) + for _, item in self.pathmapper.items(): + if src == item.resolved: + os.symlink(item.target, tgt) + break + stageFiles(generatemapper, linkoutdir) if self.stdin: - stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb") + stdin = open(self.pathmapper.reversemap(self.stdin)[1], "rb") else: stdin = subprocess.PIPE @@ -198,7 +202,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, else: stdout = sys.stderr - sp = subprocess.Popen([str(x) for x in runtime + self.command_line], + sp = subprocess.Popen([unicode(x).encode('utf-8') for x in runtime + self.command_line], shell=False, close_fds=True, stdin=stdin, @@ -232,13 +236,14 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, else: processStatus = "permanentFail" - for t in self.generatefiles: - if isinstance(self.generatefiles[t], dict): - src = cast(dict, self.generatefiles[t])["path"] - dst = os.path.join(self.outdir, t) - if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir: - os.remove(dst) - os.symlink(self.pathmapper.reversemap(src)[1], dst) + if self.generatefiles["listing"]: + def linkoutdir(src, tgt): + # Need to make the link to the staged file (may be inside + # the container) + if os.path.exists(tgt) and os.path.islink(tgt): + os.remove(tgt) + os.symlink(src, tgt) + stageFiles(generatemapper, linkoutdir) outputs = self.collect_outputs(self.outdir) @@ -266,10 +271,14 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, self.output_callback(outputs, processStatus) + if self.stagedir and os.path.exists(self.stagedir): + _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) + shutil.rmtree(self.stagedir, True) + if rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True) - if move_outputs and empty_subtree(self.outdir): + if move_outputs == "move" and empty_subtree(self.outdir): _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir) shutil.rmtree(self.outdir, True) diff --git a/cwltool/main.py b/cwltool/main.py index b86995115..1bd3f14a7 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1,31 +1,37 @@ #!/usr/bin/env python -from . import draft2tool import argparse -from schema_salad.ref_resolver import Loader import string import json import os import sys import logging import copy -from . import workflow -from .errors import WorkflowException, UnsupportedRequirement -from . import process -from .cwlrdf import printrdf, printdot -from .process import shortname, Process -from .load_tool import fetch_document, validate_document, make_tool -import schema_salad.validate as validate import tempfile -import schema_salad.jsonld_context -import schema_salad.makedoc import ruamel.yaml as yaml import urlparse +import hashlib import pkg_resources # part of setuptools +import random +import functools + import rdflib -import hashlib from typing import Union, Any, cast, Callable, Dict, Tuple, IO +from schema_salad.ref_resolver import Loader +import schema_salad.validate as validate +import schema_salad.jsonld_context +import schema_salad.makedoc + +from . import workflow +from .errors import WorkflowException, UnsupportedRequirement +from .cwlrdf import printrdf, printdot +from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps +from .load_tool import fetch_document, validate_document, make_tool +from . import draft2tool +from .builder import adjustFileObjs, adjustDirObjs +from .stdfsaccess import StdFsAccess + _logger = logging.getLogger("cwltool") defaultStreamHandler = logging.StreamHandler() @@ -81,14 +87,18 @@ def arg_parser(): # type: () -> argparse.ArgumentParser dest="rm_tmpdir") exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument("--move-outputs", action="store_true", default=True, + exgroup.add_argument("--move-outputs", action="store_const", const="move", default="move", help="Move output files to the workflow output directory and delete intermediate output directories (default).", dest="move_outputs") - exgroup.add_argument("--leave-outputs", action="store_false", default=True, + exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move", help="Leave output files in intermediate output directories.", dest="move_outputs") + exgroup.add_argument("--copy-outputs", action="store_const", const="copy", default="move", + help="Copy output files to the workflow output directory, don't delete intermediate output directories.", + dest="move_outputs") + exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--enable-pull", default=True, action="store_true", help="Try to pull Docker images", dest="enable_pull") @@ -167,12 +177,10 @@ def output_callback(out, processStatus): if "basedir" not in kwargs: raise WorkflowException("Must provide 'basedir' in kwargs") - if kwargs.get("outdir"): - pass - elif kwargs.get("dry_run"): - kwargs["outdir"] = "/tmp" - else: - kwargs["outdir"] = tempfile.mkdtemp() + output_dirs = set() + finaloutdir = kwargs.get("outdir") + kwargs["outdir"] = tempfile.mkdtemp() + output_dirs.add(kwargs["outdir"]) jobReqs = None if "cwl:requirements" in job_order_object: @@ -188,35 +196,32 @@ def output_callback(out, processStatus): output_callback, **kwargs) - if kwargs.get("conformance_test"): - job = jobiter.next() - a = {"args": job.command_line} - if job.stdin: - a["stdin"] = job.pathmapper.mapper(job.stdin)[1] - if job.stderr: - a["stderr"] = job.stderr - if job.stdout: - a["stdout"] = job.stdout - if job.generatefiles: - a["createfiles"] = job.generatefiles - return a - else: - try: - for r in jobiter: - if r: - r.run(**kwargs) - else: - raise WorkflowException("Workflow cannot make any more progress.") - except WorkflowException: - raise - except Exception as e: - _logger.exception("Got workflow error") - raise WorkflowException(unicode(e)) + try: + for r in jobiter: + if r.outdir: + output_dirs.add(r.outdir) + + if r: + r.run(**kwargs) + else: + raise WorkflowException("Workflow cannot make any more progress.") + except WorkflowException: + raise + except Exception as e: + _logger.exception("Got workflow error") + raise WorkflowException(unicode(e)) - if final_status[0] != "success": - raise WorkflowException(u"Process status is %s" % (final_status)) + if final_status[0] != "success": + raise WorkflowException(u"Process status is %s" % (final_status)) - return final_output[0] + if final_output[0] and finaloutdir: + final_output[0] = relocateOutputs(final_output[0], finaloutdir, + output_dirs, kwargs.get("move_outputs")) + + if kwargs.get("rm_tmpdir"): + cleanIntermediate(output_dirs) + + return final_output[0] class FileAction(argparse.Action): @@ -229,7 +234,20 @@ def __init__(self, option_strings, dest, nargs=None, **kwargs): def __call__(self, parser, namespace, values, option_string=None): # type: (argparse.ArgumentParser, argparse.Namespace, str, Any) -> None - setattr(namespace, self.dest, {"class": "File", "path": values}) + setattr(namespace, self.dest, {"class": "File", "location": "file://%s" % os.path.abspath(values)}) + + +class DirectoryAction(argparse.Action): + + def __init__(self, option_strings, dest, nargs=None, **kwargs): + # type: (List[str], str, Any, **Any) -> None + if nargs is not None: + raise ValueError("nargs not allowed") + super(DirectoryAction, self).__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + # type: (argparse.ArgumentParser, argparse.Namespace, str, Any) -> None + setattr(namespace, self.dest, {"class": "Directory", "location": "file://%s" % os.path.abspath(values)}) class FileAppendAction(argparse.Action): @@ -246,7 +264,7 @@ def __call__(self, parser, namespace, values, option_string=None): if not g: g = [] setattr(namespace, self.dest, g) - g.append({"class": "File", "path": values}) + g.append({"class": "File", "location": "file://%s" % os.path.abspath(values)}) def generate_parser(toolparser, tool, namemap): @@ -282,6 +300,8 @@ def generate_parser(toolparser, tool, namemap): if inptype == "File": action = cast(argparse.Action, FileAction) + elif inptype == "Directory": + action = cast(argparse.Action, DirectoryAction) elif isinstance(inptype, dict) and inptype["type"] == "array": if inptype["items"] == "File": action = cast(argparse.Action, FileAppendAction) @@ -328,6 +348,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, else: jobloaderctx = { u"path": {u"@type": u"@id"}, + u"location": {u"@type": u"@id"}, u"format": {u"@type": u"@id"}, u"id": u"@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) @@ -396,6 +417,15 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, basedir=u"file://%s/" % input_basedir) return 0 + def pathToLoc(p): + if "location" not in p: + p["location"] = p["path"] + del p["path"] + + adjustDirObjs(job_order_object, pathToLoc) + adjustFileObjs(job_order_object, pathToLoc) + adjustDirObjs(job_order_object, functools.partial(getListing, StdFsAccess(input_basedir))) + if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: @@ -404,32 +434,34 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, return (job_order_object, input_basedir) -def printdeps(obj, document_loader, stdout, relative_deps, basedir=None): +def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None): # type: (Dict[unicode, Any], Loader, IO[Any], bool, str) -> None deps = {"class": "File", - "path": obj.get("id", "#")} + "location": uri} def loadref(b, u): return document_loader.resolve_ref(u, base_url=b)[0] - sf = process.scandeps(basedir if basedir else obj["id"], obj, + sf = scandeps(basedir if basedir else uri, obj, set(("$import", "run")), - set(("$include", "$schemas", "path")), loadref) + set(("$include", "$schemas", "path", "location")), loadref) if sf: deps["secondaryFiles"] = sf if relative_deps: if relative_deps == "primary": - base = basedir if basedir else os.path.dirname(obj["id"]) + base = basedir if basedir else os.path.dirname(uri) elif relative_deps == "cwd": base = "file://" + os.getcwd() else: raise Exception(u"Unknown relative_deps %s" % relative_deps) - def makeRelative(u): + def makeRelative(ob): + u = ob.get("location", ob.get("path")) if ":" in u.split("/")[0] and not u.startswith("file://"): - return u - return os.path.relpath(u, base) - process.adjustFiles(deps, makeRelative) + pass + else: + ob["location"] = os.path.relpath(u, base) + adjustFileObjs(deps, makeRelative) stdout.write(json.dumps(deps, indent=4)) @@ -473,7 +505,7 @@ def print_pack(document_loader, processobj, uri, metadata): def loadref(b, u): # type: (unicode, unicode) -> Union[Dict, List, unicode] return document_loader.resolve_ref(u, base_url=b)[0] - deps = process.scandeps(uri, processobj, + deps = scandeps(uri, processobj, set(("run",)), set(), loadref) fdeps = set((uri,)) @@ -583,7 +615,7 @@ def main(argsl=None, document_loader, workflowobj, uri = fetch_document(args.workflow) if args.print_deps: - printdeps(workflowobj, document_loader, stdout, args.relative_deps) + printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri) return 0 document_loader, avsc_names, processobj, metadata, uri \ @@ -654,7 +686,8 @@ def main(argsl=None, if args.cachedir: setattr(args, 'cachedir', os.path.abspath(args.cachedir)) - setattr(args, 'move_outputs', False) + if args.move_outputs == "move": + setattr(args, 'move_outputs', "copy") try: setattr(args, 'tmp_outdir_prefix', @@ -666,8 +699,16 @@ def main(argsl=None, makeTool=makeTool, select_resources=selectResources, **vars(args)) + # This is the workflow output, it needs to be written if out is not None: + def locToPath(p): + if p["location"].startswith("file://"): + p["path"] = p["location"][7:] + + adjustDirObjs(out, locToPath) + adjustFileObjs(out, locToPath) + if isinstance(out, basestring): stdout.write(out) else: diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 31b0be8bc..8455d648a 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -2,10 +2,49 @@ import random import logging import stat +import collections from typing import Tuple, Set, Union, Any _logger = logging.getLogger("cwltool") +MapperEnt = collections.namedtuple("MapperEnt", ("resolved", "target", "type")) + +def adjustFiles(rec, op): # type: (Any, Callable[..., Any]) -> None + """Apply a mapping function to each File path in the object `rec`.""" + + if isinstance(rec, dict): + if rec.get("class") == "File": + rec["path"] = op(rec["path"]) + for d in rec: + adjustFiles(rec[d], op) + if isinstance(rec, list): + for d in rec: + adjustFiles(d, op) + +def adjustFileObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None + """Apply an update function to each File object in the object `rec`.""" + + if isinstance(rec, dict): + if rec.get("class") == "File": + op(rec) + for d in rec: + adjustFileObjs(rec[d], op) + if isinstance(rec, list): + for d in rec: + adjustFileObjs(d, op) + +def adjustDirObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None + """Apply an update function to each Directory object in the object `rec`.""" + + if isinstance(rec, dict): + if rec.get("class") == "Directory": + op(rec) + for d in rec: + adjustDirObjs(rec[d], op) + if isinstance(rec, list): + for d in rec: + adjustDirObjs(d, op) + def abspath(src, basedir): # type: (unicode, unicode) -> unicode if src.startswith(u"file://"): @@ -20,22 +59,83 @@ class PathMapper(object): """Mapping of files from relative path provided in the file to a tuple of (absolute local path, absolute container path)""" - def __init__(self, referenced_files, basedir): - # type: (Set[unicode], unicode) -> None + def __init__(self, referenced_files, basedir, stagedir, separateDirs=True): + # type: (Set[Any], unicode, unicode) -> None self._pathmap = {} # type: Dict[unicode, Tuple[unicode, unicode]] + self.stagedir = stagedir + self.separateDirs = separateDirs self.setup(referenced_files, basedir) + def visitlisting(self, listing, stagedir, basedir): + for ld in listing: + if "entryname" in ld: + tgt = os.path.join(stagedir, ld["entryname"]) + if isinstance(ld["entry"], (str, unicode)): + self._pathmap[str(id(ld["entry"]))] = MapperEnt(ld["entry"], tgt, "CreateFile") + else: + if ld["entry"]["class"] == "Directory": + self.visit(ld["entry"], tgt, basedir, copy=ld.get("writable", False)) + else: + self.visit(ld["entry"], stagedir, basedir, entryname=ld["entryname"], copy=ld.get("writable", False)) + #ab = ld["entry"]["location"] + #if ab.startswith("file://"): + # ab = ab[7:] + #self._pathmap[ld["entry"]["location"]] = MapperEnt(ab, tgt, ld["entry"]["class"]) + elif ld.get("class") == "File": + self.visit(ld, stagedir, basedir, copy=ld.get("writable", False)) + + def visit(self, obj, stagedir, basedir, entryname=None, copy=False): + if obj["class"] == "Directory": + if "location" in obj: + self._pathmap[obj["location"]] = MapperEnt(obj["location"], stagedir, "Directory") + else: + self._pathmap[str(id(obj))] = MapperEnt(str(id(obj)), stagedir, "Directory") + self.visitlisting(obj.get("listing", []), stagedir, basedir) + elif obj["class"] == "File": + path = obj["location"] + if path in self._pathmap: + return + ab = abspath(path, basedir) + if entryname: + tgt = os.path.join(stagedir, entryname) + else: + tgt = os.path.join(stagedir, os.path.basename(path)) + if copy: + self._pathmap[path] = MapperEnt(ab, tgt, "WritableFile") + else: + self._pathmap[path] = MapperEnt(ab, tgt, "File") + self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir) + def setup(self, referenced_files, basedir): - # type: (Set[unicode], unicode) -> None - for src in referenced_files: - ab = abspath(src, basedir) - self._pathmap[src] = (ab, ab) + # type: (Set[Any], unicode) -> None + + # Go through each file and set the target to its own directory along + # with any secondary files. + stagedir = self.stagedir + for fob in referenced_files: + if self.separateDirs: + stagedir = os.path.join(self.stagedir, "stg%x" % random.randint(1, 1000000000)) + self.visit(fob, stagedir, basedir) + + # Dereference symbolic links + for path, (ab, tgt, type) in self._pathmap.items(): + if type != "File": # or not os.path.exists(ab): + continue + deref = ab + st = os.lstat(deref) + while stat.S_ISLNK(st.st_mode): + rl = os.readlink(deref) + deref = rl if os.path.isabs(rl) else os.path.join( + os.path.dirname(deref), rl) + st = os.lstat(deref) + + self._pathmap[path] = MapperEnt(deref, tgt, "File") def mapper(self, src): # type: (unicode) -> Tuple[unicode, unicode] if u"#" in src: i = src.index(u"#") p = self._pathmap[src[:i]] - return (p[0], p[1] + src[i:]) + return (p.resolved, p.target + src[i:]) else: return self._pathmap[src] @@ -50,58 +150,3 @@ def reversemap(self, target): # type: (unicode) -> Tuple[unicode, unicode] if v[1] == target: return (k, v[0]) return None - - -class DockerPathMapper(PathMapper): - - def __init__(self, referenced_files, basedir): - # type: (Set[unicode], unicode) -> None - self.dirs = {} # type: Dict[unicode, Union[bool, unicode]] - super(DockerPathMapper, self).__init__(referenced_files, basedir) - - def setup(self, referenced_files, basedir): - for src in referenced_files: - ab = abspath(src, basedir) - dirn, fn = os.path.split(ab) - - subdir = False - for d in self.dirs: - if dirn.startswith(d): - subdir = True - break - - if not subdir: - for d in list(self.dirs): - if d.startswith(dirn): - # 'dirn' is a parent of 'd' - del self.dirs[d] - self.dirs[dirn] = True - - prefix = u"job" + str(random.randint(1, 1000000000)) + u"_" - - names = set() # type: Set[unicode] - for d in self.dirs: - name = os.path.join(u"/var/lib/cwl", prefix + os.path.basename(d)) - i = 1 - while name in names: - i += 1 - name = os.path.join(u"/var/lib/cwl", - prefix + os.path.basename(d) + unicode(i)) - names.add(name) - self.dirs[d] = name - - for src in referenced_files: - ab = abspath(src, basedir) - - deref = ab - st = os.lstat(deref) - while stat.S_ISLNK(st.st_mode): - rl = os.readlink(deref) - deref = rl if os.path.isabs(rl) else os.path.join( - os.path.dirname(deref), rl) - st = os.lstat(deref) - - for d in self.dirs: - if ab.startswith(d): - self._pathmap[src] = (deref, os.path.join( - self.dirs[d], ab[len(d)+1:])) diff --git a/cwltool/process.py b/cwltool/process.py index 98e35cf52..ea708c36c 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1,45 +1,50 @@ -import abc -import avro.schema + import os import json -import schema_salad.validate as validate import copy import logging import pprint -from .utils import aslist, get_feature -import schema_salad.schema -from schema_salad.ref_resolver import Loader -import urlparse -import pprint -from pkg_resources import resource_stream import stat -from .builder import Builder, adjustFileObjs import tempfile import glob -from .errors import WorkflowException, UnsupportedRequirement -from .pathmapper import abspath +import urlparse +import pprint +from collections import Iterable +import errno +import random +import shutil + +import abc +import schema_salad.validate as validate +import schema_salad.schema +from schema_salad.ref_resolver import Loader +import avro.schema from typing import (Any, AnyStr, Callable, cast, Dict, List, Generator, IO, Tuple, Union) -from collections import Iterable from rdflib import URIRef from rdflib.namespace import RDFS, OWL -from .stdfsaccess import StdFsAccess -import errno from rdflib import Graph +from pkg_resources import resource_stream + +from .utils import aslist, get_feature +from .stdfsaccess import StdFsAccess +from .builder import Builder, adjustFileObjs, adjustDirObjs +from .errors import WorkflowException, UnsupportedRequirement +from .pathmapper import PathMapper, abspath _logger = logging.getLogger("cwltool") supportedProcessRequirements = ["DockerRequirement", "SchemaDefRequirement", "EnvVarRequirement", - "CreateFileRequirement", "ScatterFeatureRequirement", "SubworkflowFeatureRequirement", "MultipleInputFeatureRequirement", "InlineJavascriptRequirement", "ShellCommandRequirement", "StepInputExpressionRequirement", - "ResourceRequirement"] + "ResourceRequirement", + "InitialWorkDirRequirement"] cwl_files = ("Workflow.yml", "CommandLineTool.yml", @@ -74,6 +79,8 @@ SCHEMA_CACHE = {} # type: Dict[str, Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[unicode, Any], Loader]] SCHEMA_FILE = None # type: Dict[unicode, Any] +SCHEMA_DIR = None # type: Dict[unicode, Any] +SCHEMA_DIRENT = None # type: Dict[unicode, Any] SCHEMA_ANY = None # type: Dict[unicode, Any] def get_schema(version): @@ -106,19 +113,13 @@ def get_schema(version): SCHEMA_CACHE[version] = schema_salad.schema.load_schema( "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache) - global SCHEMA_FILE, SCHEMA_ANY # pylint: disable=global-statement - SCHEMA_FILE = cast(Dict[unicode, Any], - SCHEMA_CACHE[version][3].idx["https://w3id.org/cwl/cwl#File"]) - SCHEMA_ANY = cast(Dict[unicode, Any], - SCHEMA_CACHE[version][3].idx["https://w3id.org/cwl/salad#Any"]) - return SCHEMA_CACHE[version] def shortname(inputid): # type: (unicode) -> unicode d = urlparse.urlparse(inputid) if d.fragment: - return d.fragment.split(u"/")[-1].split(u".")[-1] + return d.fragment.split(u"/")[-1] else: return d.path.split(u"/")[-1] @@ -135,18 +136,6 @@ def checkRequirements(rec, supportedProcessRequirements): for d in rec: checkRequirements(d, supportedProcessRequirements) -def adjustFiles(rec, op): # type: (Any, Callable[..., Any]) -> None - """Apply a mapping function to each File path in the object `rec`.""" - - if isinstance(rec, dict): - if rec.get("class") == "File": - rec["path"] = op(rec["path"]) - for d in rec: - adjustFiles(rec[d], op) - if isinstance(rec, list): - for d in rec: - adjustFiles(d, op) - def adjustFilesWithSecondary(rec, op, primary=None): """Apply a mapping function to each File path in the object `rec`, propagating the primary file associated with a group of secondary files. @@ -164,6 +153,80 @@ def adjustFilesWithSecondary(rec, op, primary=None): for d in rec: adjustFilesWithSecondary(d, op, primary) +def getListing(fs_access, rec): + if "listing" not in rec: + listing = [] + loc = rec["location"] + for ld in fs_access.listdir(loc): + if fs_access.isdir(ld): + ent = {"class": "Directory", + "location": ld} + getListing(fs_access, ent) + listing.append({"entryname": os.path.basename(ld), + "entry": ent}) + else: + listing.append({"entryname": os.path.basename(ld), + "entry": {"class": "File", "location": ld}}) + rec["listing"] = listing + +def stageFiles(pm, stageFunc): + for f, p in pm.items(): + if not os.path.exists(os.path.dirname(p.target)): + os.makedirs(os.path.dirname(p.target), 0755) + if p.type == "File": + stageFunc(p.resolved, p.target) + elif p.type == "WritableFile": + shutil.copy(p.resolved, p.target) + elif p.type == "CreateFile": + with open(p.target, "w") as n: + n.write(p.resolved.encode("utf-8")) + +def collectFilesAndDirs(obj, out): + if isinstance(obj, dict): + if obj.get("class") in ("File", "Directory"): + out.append(obj) + else: + for v in obj.values(): + collectFilesAndDirs(v, out) + if isinstance(obj, list): + for l in obj: + collectFilesAndDirs(l, out) + +def relocateOutputs(outputObj, outdir, output_dirs, action): + if action not in ("move", "copy"): + return outputObj + + def moveIt(src, dst): + for a in output_dirs: + if src.startswith(a): + if action == "move": + _logger.debug("Moving %s to %s", src, dst) + shutil.move(src, dst) + elif action == "copy": + _logger.debug("Copying %s to %s", src, dst) + shutil.copy(src, dst) + + outfiles = [] + collectFilesAndDirs(outputObj, outfiles) + pm = PathMapper(outfiles, "", outdir, separateDirs=False) + stageFiles(pm, moveIt) + + def _check_adjust(f): + f["location"] = "file://" + pm.mapper(f["location"])[1] + return f + + adjustFileObjs(outputObj, _check_adjust) + adjustDirObjs(outputObj, _check_adjust) + + return outputObj + +def cleanIntermediate(output_dirs): + for a in output_dirs: + if os.path.exists(a) and empty_subtree(a): + _logger.debug(u"Removing intermediate output directory %s", a) + shutil.rmtree(a, True) + + def formatSubclassOf(fmt, cls, ontology, visited): # type: (str, str, Graph, Set[str]) -> bool """Determine if `fmt` is a subclass of `cls`.""" @@ -244,8 +307,21 @@ def __init__(self, toolpath_object, **kwargs): # type: (Dict[unicode, Any], **Any) -> None self.metadata = kwargs.get("metadata", {}) # type: Dict[str,Any] self.names = None # type: avro.schema.Names - names = schema_salad.schema.make_avro_schema( - [SCHEMA_FILE, SCHEMA_ANY], schema_salad.ref_resolver.Loader({}))[0] + + if SCHEMA_FILE is None: + global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_DIRENT, SCHEMA_ANY # pylint: disable=global-statement + get_schema("draft-4") + SCHEMA_ANY = cast(Dict[unicode, Any], + SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/salad#Any"]) + SCHEMA_FILE = cast(Dict[unicode, Any], + SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#File"]) + SCHEMA_DIR = cast(Dict[unicode, Any], + SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#Directory"]) + SCHEMA_DIRENT = cast(Dict[unicode, Any], + SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#Dirent"]) + + names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_DIRENT, SCHEMA_ANY], + schema_salad.ref_resolver.Loader({}))[0] if isinstance(names, avro.schema.SchemaParseException): raise names else: @@ -333,13 +409,19 @@ def _init_job(self, joborder, **kwargs): builder.resources = {} builder.timeout = kwargs.get("eval_timeout") - dockerReq, _ = self.get_requirement("DockerRequirement") + dockerReq, is_req = self.get_requirement("DockerRequirement") + + if dockerReq and is_req and not kwargs.get("use_container"): + raise WorkflowException("Document has DockerRequirement under 'requirements' but use_container is false. DockerRequirement must be under 'hints' or use_container must be true.") + if dockerReq and kwargs.get("use_container"): builder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl" builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp" + builder.stagedir = kwargs.get("docker_stagedir") or "/var/lib/cwl" else: builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp() builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp() + builder.stagedir = kwargs.get("stagedir") or tempfile.mkdtemp() builder.fs_access = kwargs.get("fs_access") or StdFsAccess(kwargs["basedir"]) @@ -369,6 +451,12 @@ def _init_job(self, joborder, **kwargs): a["do_eval"] = a["valueFrom"] a["valueFrom"] = None builder.bindings.append(a) + elif ("$(" in a) or ("${" in a): + builder.bindings.append({ + "position": [0, i], + "do_eval": a, + "valueFrom": None + }) else: builder.bindings.append({ "position": [0, i], @@ -475,6 +563,41 @@ def uniquename(stem): # type: (unicode) -> unicode _names.add(u) return u +def nestdir(base, deps): + dirname = os.path.dirname(base) + "/" + subid = deps["location"] + if subid.startswith(dirname): + s2 = subid[len(dirname):] + sp = s2.split('/') + sp.pop() + while sp: + nx = sp.pop() + deps = { + "entryname": nx, + "entry": { + "class": "Directory", + "listing": [deps] + } + } + return deps + +def mergedirs(listing): + r = [] + ents = {} + for e in listing: + if "entryname" in e: + if e["entryname"] not in ents: + ents[e["entryname"]] = e + elif e["entry"]["class"] == "Directory": + ents[e["entryname"]]["entry"]["listing"].extend(e["entry"]["listing"]) + else: + r.append(e) + for e in ents.itervalues(): + if e["entry"]["class"] == "Directory": + e["entry"]["listing"] = mergedirs(e["entry"]["listing"]) + r.extend(ents.itervalues()) + return r + def scandeps(base, doc, reffields, urlfields, loadref): # type: (unicode, Any, Set[str], Set[str], Callable[[unicode, str], Any]) -> List[Dict[str, str]] r = [] @@ -485,7 +608,7 @@ def scandeps(base, doc, reffields, urlfields, loadref): if base != df: r.append({ "class": "File", - "path": df + "location": df }) base = df @@ -499,21 +622,26 @@ def scandeps(base, doc, reffields, urlfields, loadref): subid = urlparse.urljoin(base, u) deps = { "class": "File", - "path": subid - } # type: Dict[str, Any] + "location": subid + } # type: Dict[str, Any] sf = scandeps(subid, sub, reffields, urlfields, loadref) if sf: deps["secondaryFiles"] = sf + deps = nestdir(base, deps) r.append(deps) elif k in urlfields: for u in aslist(v): - r.append({ + deps = { "class": "File", - "path": urlparse.urljoin(base, u) - }) + "location": urlparse.urljoin(base, u) + } + deps = nestdir(base, deps) + r.append(deps) else: r.extend(scandeps(base, v, reffields, urlfields, loadref)) elif isinstance(doc, list): for d in doc: r.extend(scandeps(base, d, reffields, urlfields, loadref)) + + r = mergedirs(r) return r diff --git a/cwltool/schemas/draft-3/examples/arguments.cwl b/cwltool/schemas/draft-3/examples/arguments.cwl index 862540be2..23e91ebdd 100644 --- a/cwltool/schemas/draft-3/examples/arguments.cwl +++ b/cwltool/schemas/draft-3/examples/arguments.cwl @@ -1,5 +1,6 @@ cwlVersion: cwl:draft-3 class: CommandLineTool +label: Example trivial wrapper for Java 7 compiler baseCommand: javac hints: - class: DockerRequirement diff --git a/cwltool/schemas/draft-4/CommandLineTool.yml b/cwltool/schemas/draft-4/CommandLineTool.yml index cec579407..f33dadfc4 100644 --- a/cwltool/schemas/draft-4/CommandLineTool.yml +++ b/cwltool/schemas/draft-4/CommandLineTool.yml @@ -79,30 +79,6 @@ $graph: - {$include: concepts.md} - {$include: invocation.md} -- type: record - name: FileDef - doc: | - Define a file that must be placed in the designated output directory - prior to executing the command line tool. May be the result of executing - an expression, such as building a configuration file from a template. - fields: - - name: "filename" - type: ["string", "#Expression"] - doc: "The name of the file to create in the output directory." - - name: "fileContent" - type: ["string", "#Expression"] - doc: | - If the value is a string literal or an expression which evaluates to a - string, a new file must be created with the string as the file contents. - - If the value is an expression that evaluates to a File object, this - indicates the referenced file should be added to the designated output - directory prior to executing the tool. - - Files added in this way may be read-only, and may be provided - by bind mounts or file system links to avoid - unnecessary copying of the input file. - - type: record name: EnvironmentDef @@ -248,17 +224,6 @@ $graph: the File objects must include up to the first 64 KiB of file contents in the `contents` field. -- name: initialWorkDir - type: record - doc: | - Setup a working directory based on a number of input files (and/or directories) - fields: - - name: dirDef - type: - type: "array" - items: [ "#FileDef", "#File", "#Directory" ] - doc: list of files and/or directories - - name: CommandInputRecordField type: record @@ -499,7 +464,7 @@ $graph: type: - "null" - type: array - items: [string, "#CommandLineBinding"] + items: [string, "#Expression", "#CommandLineBinding"] jsonldPredicate: "_id": "cwl:arguments" "_container": "@list" @@ -626,21 +591,72 @@ $graph: Docker container. - -- name: CreateFileRequirement +- name: DirentExt type: record - extends: "#ProcessRequirement" doc: | - Define a list of files that must be created by the workflow - platform in the designated output directory prior to executing the command - line tool. See `FileDef` for details. + Define a file or subdirectory that must be placed in the designated output + directory prior to executing the command line tool. May be the result of + executing an expression, such as building a configuration file from a + template. fields: - - name: fileDef + - name: entryname + type: [string, Expression] + jsonldPredicate: + _id: cwl:entryname + doc: | + The name of the file or subdirectory to create in the output directory. + - name: entry + type: [string, Expression] + jsonldPredicate: + _id: cwl:entry + doc: | + If the value is a string literal or an expression which evaluates to a + string, a new file must be created with the string as the file contents. + + If the value is an expression that evaluates to a `File` object, this + indicates the referenced file should be added to the designated output + directory prior to executing the tool. + + If the value is an expression that evaluates to a `Dirent` object, this + indicates that the File or Directory in `entry` should be added to the + designated output directory with the name in `entryname`. + + If `writable` is false, the file may be made available using a bind + mount or file system link to avoid unnecessary copying of the input + file. + - name: writable + type: boolean? + doc: | + If true, the file or directory must be writable by the tool. Changes + to the file or directory must be isolated and not visible by any other + CommandLineTool process. This may be implemented by making a copy of + the original file or directory. Default false (files and directories + read-only by default). + +- name: InitialWorkDirRequirement + type: record + extends: ProcessRequirement + doc: + Define a list of files and subdirectories that must be created by the + workflow platform in the designated output directory prior to executing the + command line tool. + fields: + - name: listing type: - type: "array" - items: "#FileDef" - doc: The list of files. + - type: array + items: [File, DirentExt] + - string + - Expression + jsonldPredicate: + _id: "cwl:listing" + mapSubject: entryname + mapPredicate: entry + doc: | + The list of files or subdirectories that must be placed in the + designated output directory prior to executing the command line tool. + May be an expression. If so, the expression return value must validate + as `{type: array, items: [File, Dirent]}`. - name: EnvVarRequirement type: record diff --git a/cwltool/schemas/draft-4/Process.yml b/cwltool/schemas/draft-4/Process.yml index 319ff27f0..351bab9d2 100644 --- a/cwltool/schemas/draft-4/Process.yml +++ b/cwltool/schemas/draft-4/Process.yml @@ -33,15 +33,18 @@ $graph: - cwl:draft-3 - cwl:draft-4.dev1 - cwl:draft-4.dev2 + - cwl:draft-4.dev3 - name: CWLType type: enum extends: "sld:PrimitiveType" symbols: - cwl:File + - cwl:Directory doc: - "Extends primitive types with the concept of a file as a first class type." - "File: A File object" + - "Directory: A Directory object" - name: File type: record @@ -58,20 +61,87 @@ $graph: symbols: - cwl:File jsonldPredicate: - "_id": "@type" - "_type": "@vocab" + _id: "@type" + _type: "@vocab" doc: Must be `File` to indicate this object describes a file. - - name: path + - name: location type: string - doc: The path to the file. + doc: | + A URI that identifies the file resource. This may be a relative + reference, in which case it must be resolved using the base URI of the + document. The location may refer to a local or remote resource; the + implementation must use the URI to retrieve file content. If an + implementation is unable to retrieve the file content stored at a + remote resource (due to unsupported protocol, access denied, or other + issue) it must signal an error. + + If the `path` field is provided but the `location` field is not, an + implementation may assign the value of the `path` field to `location`, + then follow the rules above. + jsonldPredicate: + _id: "@id" + _type: "@id" + - name: path + type: string? + doc: | + The local path where the File is made available prior to executing a + CommandLineTool. This must be set by the implementation. This field + must not be used in any other context. The command line tool being + executed must be able to to access the file at `path` using the POSIX + `open(2)` syscall. jsonldPredicate: "_id": "cwl:path" "_type": "@id" + - name: basename + type: string? + doc: | + The base name of the file, that is, the path component following the + final slash in the path. + + The implementation must set this field based on the value of `path` + prior to evaluating parameter references or expressions in a + CommandLineTool document. This field must not be used in any other + context. + - name: dirname + type: string? + doc: | + The name of the directory containing file, that is, the path leading up + to the final slash in the path such that `dirname + '/' + basename = + path`. + + The implementation must set this field based on the value of `path` + prior to evaluating parameter references or expressions in a + CommandLineTool document. This field must not be used in any other + context. + - name: nameroot + type: string? + doc: | + The basename root such that `nameroot + nameext == basename`, and + `nameext` is empty or begins with a period and contains at most one + period. Leading periods on the basename are ignored; a basename of + `.cshrc` will have a nameroot of `.cshrc`. + + The implementation must set this field based on the value of `path` + prior to evaluating parameter references or expressions in a + CommandLineTool document. This field must not be used in any other + context. + - name: nameext + type: string? + doc: | + The basename extension such that `nameroot + nameext == basename`, and + `nameext` is empty or begins with a period and contains at most one + period. Leading periods on the basename are ignored; a basename of + `.cshrc` will have an empty `nameext`. + + The implementation must set this field based on the value of `path` + prior to evaluating parameter references or expressions in a + CommandLineTool document. This field must not be used in any other + context. - name: checksum type: ["null", string] doc: | Optional hash code for validating file integrity. Currently must be in the form - "sha1$ + hexidecimal string" using the SHA-1 algorithm. + "sha1$ + hexadecimal string" using the SHA-1 algorithm. - name: size type: ["null", long] doc: Optional file size. @@ -79,7 +149,7 @@ $graph: type: - "null" - type: array - items: "#File" + items: [File, Dirent] jsonldPredicate: "cwl:secondaryFiles" doc: | A list of additional files that are associated with the primary file @@ -111,12 +181,24 @@ $graph: runtime may perform exact file format matches. +- name: Dirent + type: record + fields: + - name: entryname + type: string + jsonldPredicate: + "_id": cwl:entryname + - name: entry + type: [File, Directory] + jsonldPredicate: + "_id": cwl:entry + - name: Directory type: record docParent: "#CWLType" doc: | Represents a directory to present to a command line tool. This could be a virtual - directory, made of files assembled from a number of concrete directories. + directory, made of files assembled from multiple locations. fields: - name: class type: @@ -125,18 +207,48 @@ $graph: symbols: - cwl:Directory jsonldPredicate: - "_id": "@type" - "_type": "@vocab" + _id: "@type" + _type: "@vocab" doc: Must be `Directory` to indicate this object describes a Directory. - - name: path + - name: location type: string - doc: The path to the directory. + doc: | + A URI that identifies the directory resource. This may be a relative + reference, in which case it must be resolved using the base URI of the + document. The location may refer to a local or remote resource. If + the `listing` field is not set, the implementation must use the + location URI to retrieve directory listing. If an implementation is + unable to retrieve the directory listing stored at a remote resource (due to + unsupported protocol, access denied, or other issue) it must signal an + error. + + If the `path` field is provided but the `location` field is not, an + implementation may assign the value of the `path` field to `location`, + then follow the rules above. jsonldPredicate: - "_id": "cwl:path" - "_type": "@id" - # - name: size - # type: ["null", long] - # doc: Optional directory size. + _id: "@id" + _type: "@id" + - name: path + type: string? + doc: | + The local path where the Directory is made available prior to executing a + CommandLineTool. This must be set by the implementation. This field + must not be used in any other context. The command line tool being + executed must be able to to access the directory at `path` using the POSIX + `opendir(2)` syscall. + jsonldPredicate: + _id: "cwl:path" + _type: "@id" + - name: listing + type: + - "null" + - type: array + items: [File, Dirent] + doc: List of files or subdirectories contained in this directory + jsonldPredicate: + _id: "cwl:listing" + mapSubject: entryname + mapPredicate: entry - name: SchemaBase type: record diff --git a/cwltool/schemas/draft-4/conformance_test_draft-4.yaml b/cwltool/schemas/draft-4/conformance_test_draft-4.yaml index ae839ba0a..7769640b8 100644 --- a/cwltool/schemas/draft-4/conformance_test_draft-4.yaml +++ b/cwltool/schemas/draft-4/conformance_test_draft-4.yaml @@ -40,7 +40,7 @@ "foo": { "checksum": "sha1$63da67422622fbf9251a046d7a34b7ea0fd4fead", "class": "File", - "path": "foo.txt", + "location": "foo.txt", "size": 22 } job: draft-4/cat-job.json @@ -52,44 +52,61 @@ output_file: class: File checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b - path: output.txt + location: output.txt size: 13 tool: draft-4/cat3-tool.cwl doc: Test command execution in Docker with stdout redirection - job: draft-4/cat-job.json tool: draft-4/cat3-tool-shortcut.cwl - doc: Test command execution in Docker with stdout redirection + output: + output_file: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: + size: 13 + doc: Test command execution in Docker with simplified syntax stdout redirection - job: draft-4/cat-job.json output: output_file: class: File checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b - path: cat-out + location: cat-out size: 13 tool: draft-4/cat3-tool-mediumcut.cwl doc: Test command execution in Docker with stdout redirection -- args: [egrep] - stderr: error.txt - job: +- job: + output: { + "output_file": { + "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976", + "size": 84, + "location": "error.txt", + "class": "File" + } + } tool: draft-4/egrep-stderr.cwl doc: Test command line with stderr redirection -- args: [egrep] - job: +- job: + output: { + "output_file": { + "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976", + "size": 84, + "location": "", + "class": "File" + } + } tool: draft-4/egrep-stderr-shortcut.cwl doc: Test command line with stderr redirection, brief syntax -- args: [egrep] - stderr: std.err - output: +- output: output_file: class: File size: 84 checksum: sha1$cec7b8746a78c42060c96505887449bca0142976 - path: std.err + location: std.err job: tool: draft-4/egrep-stderr-mediumcut.cwl doc: Test command line with stderr redirection, named brief syntax @@ -99,7 +116,7 @@ output_txt: class: File checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b - path: output.txt + location: output.txt size: 13 tool: draft-4/cat4-tool.cwl doc: Test command execution in Docker with stdin and stdout redirection @@ -144,7 +161,7 @@ output: class: File checksum: sha1$631bfbac524e2d04cdcc5ec33ade827fc10b06ae - path: output + location: output size: 15 tool: draft-4/wc-tool.cwl doc: Test command execution in with stdin and stdout redirection @@ -214,7 +231,7 @@ out: class: File checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 - path: out + location: out size: 15 tool: draft-4/env-tool1.cwl doc: Test EnvVarRequirement @@ -259,7 +276,7 @@ out: class: File checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 - path: out + location: out size: 15 tool: draft-4/env-wf1.cwl doc: Test requirement priority @@ -269,7 +286,7 @@ out: class: File checksum: sha1$cdc1e84968261d6a7575b5305945471f8be199b6 - path: out + location: out size: 9 tool: draft-4/env-wf2.cwl doc: Test requirements override hints @@ -284,7 +301,7 @@ output: class: File checksum: sha1$b9214658cc453331b62c2282b772a5c063dbd284 - path: output.txt + location: output.txt size: 1111 tool: draft-4/revsort.cwl doc: Test sample workflows from the specification @@ -294,7 +311,7 @@ output_file: class: File checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b - path: output.txt + location: output.txt size: 13 tool: draft-4/cat5-tool.cwl doc: Test unknown hints are ignored. @@ -304,32 +321,32 @@ outfile: class: File checksum: sha1$e2dc9daaef945ac15f01c238ed2f1660f60909a0 - path: result.txt + location: result.txt size: 142 indexedfile: { - "path": "input.txt", + "location": "input.txt", "class": "File", "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376", "secondaryFiles": [ { - "path": "input.txt.idx1", - "class": "File" + "location": "input.txt.idx1", + "class": "File", }, { - "path": "input.idx2", - "class": "File" + "location": "input.idx2", + "class": "File", }, { - "path": "input.txt.idx3", - "class": "File" + "location": "input.txt.idx3", + "class": "File", }, { - "path": "input.txt.idx4", - "class": "File" + "location": "input.txt.idx4", + "class": "File", }, { - "path": "input.txt.idx5", - "class": "File" + "location": "input.txt.idx5", + "class": "File", } ], "size": 1111 @@ -344,7 +361,7 @@ outfile: class: File checksum: sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376 - path: fish.txt + location: fish.txt size: 1111 tool: draft-4/rename.cwl doc: | @@ -360,7 +377,7 @@ - job: draft-4/schemadef-job.json output: output: - path: output.txt + location: output.txt size: 12 class: File checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e" @@ -371,7 +388,7 @@ - job: draft-4/schemadef-job.json output: output: - path: output.txt + location: output.txt size: 12 class: File checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e" @@ -537,7 +554,7 @@ - job: draft-4/formattest-job.json output: output: - "path": "output.txt" + "location": "output.txt" "format": "http://edamontology.org/format_2330" "size": 1111 "class": "File" @@ -549,7 +566,7 @@ - job: draft-4/formattest2-job.json output: output: - "path": "output.txt" + "location": "output.txt" "format": "http://edamontology.org/format_1929" "size": 12010 "class": "File" @@ -561,7 +578,7 @@ - job: draft-4/formattest2-job.json output: output: - "path": "output.txt" + "location": "output.txt" "format": "http://edamontology.org/format_1929" "size": 12010 "class": "File" @@ -575,7 +592,7 @@ output: optional_file: null output_file: - path: output.txt + location: output.txt size: 13 class: "File" checksum: "sha1$47a013e660d408619d894b20806b1d5086aab03b" @@ -601,13 +618,13 @@ output: "orec": { "ofoo": { - "path": "foo", + "location": "foo", "size": 1111, "class": "File", "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376" }, "obar": { - "path": "bar", + "location": "bar", "size": 12010, "class": "File", "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" @@ -619,7 +636,7 @@ - job: draft-4/empty.json output: { "foo": { - "path": "foo", + "location": "foo", "class": "File" } } @@ -629,19 +646,19 @@ - job: draft-4/abc.json output: files: [{ - "path": "a", + "location": "a", "size": 0, "class": "File", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" }, { - "path": "b", + "location": "b", "size": 0, "class": "File", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" }, { - "path": "c", + "location": "c", "size": 0, "class": "File", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" @@ -676,7 +693,7 @@ - job: draft-4/conflict-job.json output: { "fileout": { - "path": "out.txt", + "location": "out.txt", "checksum": "sha1$a2d8d6e7b28295dc9977dc3bdb652ddd480995f0", "class": "File", "size": 25 @@ -684,3 +701,73 @@ } tool: "draft-4/conflict-wf.cwl#collision" doc: Test workflow two input files with same name. + +- job: draft-4/dir-job.yml + output: + "outlist": { + "size": 20, + "location": "output.txt", + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "class": "File" + } + tool: draft-4/dir.cwl + doc: Test directory input + +- job: draft-4/dir-job.yml + output: + "outlist": { + "size": 20, + "location": "output.txt", + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "class": "File" + } + tool: draft-4/dir2.cwl + doc: Test directory input in Docker + +- job: draft-4/dir3-job.yml + output: + "outdir": { + "class": "Directory", + "listing": [ + { + "entryname": "hello.txt", + "entry": { + "class": "File", + "location": "hello.txt" + } + }, + { + "entryname": "goodbye.txt", + "entry": { + "class": "File", + "location": "goodbye.txt" + } + } + ], + } + tool: draft-4/dir3.cwl + doc: Test directory input in Docker + +- job: draft-4/dir4-job.yml + output: { + "outlist": { + "checksum": "sha1$2ab6f189e84753c05a23413fbf6b6fbf4c53489f", + "size": 90, + "location": "output.txt", + "class": "File" + } + } + tool: draft-4/dir4.cwl + doc: Test directories in secondaryFiles + +- job: draft-4/dir-job.yml + output: { + "outlist": { + "checksum": "sha1$907a866a3e0b7f1fc5a2222531c5fb9063704438", + "size": 33, + "location": "output.txt", + "class": "File" + } + } + tool: draft-4/dir5.cwl + doc: Test dynamic initial work dir diff --git a/cwltool/schemas/draft-4/draft-4/binding-test.cwl b/cwltool/schemas/draft-4/draft-4/binding-test.cwl index 7e6c3c826..395c50a92 100755 --- a/cwltool/schemas/draft-4/draft-4/binding-test.cwl +++ b/cwltool/schemas/draft-4/draft-4/binding-test.cwl @@ -1,7 +1,7 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: - id: reference @@ -19,7 +19,7 @@ inputs: type: File default: class: File - path: args.py + location: args.py inputBinding: position: -1 diff --git a/cwltool/schemas/draft-4/draft-4/bwa-mem-job.json b/cwltool/schemas/draft-4/draft-4/bwa-mem-job.json index 48d7d6cfe..f3e900dea 100644 --- a/cwltool/schemas/draft-4/draft-4/bwa-mem-job.json +++ b/cwltool/schemas/draft-4/draft-4/bwa-mem-job.json @@ -1,18 +1,18 @@ { "reference": { "class": "File", - "path": "chr20.fa", + "location": "chr20.fa", "size": 123, "checksum": "sha1$hash" }, "reads": [ { "class": "File", - "path": "example_human_Illumina.pe_1.fastq" + "location": "example_human_Illumina.pe_1.fastq" }, { "class": "File", - "path": "example_human_Illumina.pe_2.fastq" + "location": "example_human_Illumina.pe_2.fastq" } ], "min_std_max_min": [ diff --git a/cwltool/schemas/draft-4/draft-4/bwa-mem-tool.cwl b/cwltool/schemas/draft-4/draft-4/bwa-mem-tool.cwl index b0722effa..63ecb885e 100755 --- a/cwltool/schemas/draft-4/draft-4/bwa-mem-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/bwa-mem-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: draft-4.dev2 +cwlVersion: draft-4.dev3 class: CommandLineTool @@ -34,7 +34,7 @@ inputs: type: File default: class: File - path: args.py + location: args.py inputBinding: position: -1 diff --git a/cwltool/schemas/draft-4/draft-4/cat-job.json b/cwltool/schemas/draft-4/draft-4/cat-job.json index 09f16ba7b..837875d92 100644 --- a/cwltool/schemas/draft-4/draft-4/cat-job.json +++ b/cwltool/schemas/draft-4/draft-4/cat-job.json @@ -1,6 +1,6 @@ { "file1": { "class": "File", - "path": "hello.txt" + "location": "hello.txt" } } diff --git a/cwltool/schemas/draft-4/draft-4/cat-n-job.json b/cwltool/schemas/draft-4/draft-4/cat-n-job.json index ee6416857..1b93b815a 100644 --- a/cwltool/schemas/draft-4/draft-4/cat-n-job.json +++ b/cwltool/schemas/draft-4/draft-4/cat-n-job.json @@ -1,7 +1,7 @@ { "file1": { "class": "File", - "path": "hello.txt" + "location": "hello.txt" }, "numbering": true } diff --git a/cwltool/schemas/draft-4/draft-4/cat1-testcli.cwl b/cwltool/schemas/draft-4/draft-4/cat1-testcli.cwl index 3bad2b662..04c3857f6 100755 --- a/cwltool/schemas/draft-4/draft-4/cat1-testcli.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat1-testcli.cwl @@ -1,7 +1,7 @@ #!/usr/bin/env cwl-runner { "class": "CommandLineTool", - "cwlVersion": "cwl:draft-4.dev2", + "cwlVersion": "cwl:draft-4.dev3", "description": "Print the contents of a file to stdout using 'cat' running in a docker container.", "inputs": [ { @@ -22,7 +22,7 @@ type: File, default: { class: File, - path: args.py + location: args.py }, inputBinding: { position: -1 diff --git a/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl index 8c7402f33..2ccde1553 100755 --- a/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: diff --git a/cwltool/schemas/draft-4/draft-4/cat2-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat2-tool.cwl index 8038f5163..8348a82f5 100755 --- a/cwltool/schemas/draft-4/draft-4/cat2-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat2-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/cat3-tool-mediumcut.cwl b/cwltool/schemas/draft-4/draft-4/cat3-tool-mediumcut.cwl index 1c29102eb..2e3f7282a 100755 --- a/cwltool/schemas/draft-4/draft-4/cat3-tool-mediumcut.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat3-tool-mediumcut.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/cat3-tool-shortcut.cwl b/cwltool/schemas/draft-4/draft-4/cat3-tool-shortcut.cwl index 4d12a4df4..04eb14d5e 100755 --- a/cwltool/schemas/draft-4/draft-4/cat3-tool-shortcut.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat3-tool-shortcut.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/cat3-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat3-tool.cwl index 56df0f6f8..f67588a9b 100755 --- a/cwltool/schemas/draft-4/draft-4/cat3-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat3-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/cat4-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat4-tool.cwl index 1fefd2cf9..733bc9167 100755 --- a/cwltool/schemas/draft-4/draft-4/cat4-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat4-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/cat5-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat5-tool.cwl index f7010e4a8..24e89d91d 100755 --- a/cwltool/schemas/draft-4/draft-4/cat5-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat5-tool.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: diff --git a/cwltool/schemas/draft-4/draft-4/conflict-wf.cwl b/cwltool/schemas/draft-4/draft-4/conflict-wf.cwl index 2e197004d..c72b27bf2 100644 --- a/cwltool/schemas/draft-4/draft-4/conflict-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/conflict-wf.cwl @@ -1,4 +1,4 @@ -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: echo class: CommandLineTool diff --git a/cwltool/schemas/draft-4/draft-4/count-lines1-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines1-wf.cwl index 96244ec2b..7c58ec84f 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines1-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines1-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: diff --git a/cwltool/schemas/draft-4/draft-4/count-lines2-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines2-wf.cwl index 65c1fc1d3..267f949ba 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines2-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines2-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: InlineJavascriptRequirement: {} diff --git a/cwltool/schemas/draft-4/draft-4/count-lines3-job.json b/cwltool/schemas/draft-4/draft-4/count-lines3-job.json index 3bdd5a322..3a93e32d4 100644 --- a/cwltool/schemas/draft-4/draft-4/count-lines3-job.json +++ b/cwltool/schemas/draft-4/draft-4/count-lines3-job.json @@ -2,11 +2,11 @@ "file1": [ { "class": "File", - "path": "whale.txt" + "location": "whale.txt" }, { "class": "File", - "path": "hello.txt" + "location": "hello.txt" } ] } diff --git a/cwltool/schemas/draft-4/draft-4/count-lines3-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines3-wf.cwl index f7cde5a31..d48cd9258 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines3-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines3-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: diff --git a/cwltool/schemas/draft-4/draft-4/count-lines4-job.json b/cwltool/schemas/draft-4/draft-4/count-lines4-job.json index dcd309a6e..1c85bea01 100644 --- a/cwltool/schemas/draft-4/draft-4/count-lines4-job.json +++ b/cwltool/schemas/draft-4/draft-4/count-lines4-job.json @@ -1,10 +1,10 @@ { "file1": { "class": "File", - "path": "whale.txt" + "location": "whale.txt" }, "file2": { "class": "File", - "path": "hello.txt" + "location": "hello.txt" } } diff --git a/cwltool/schemas/draft-4/draft-4/count-lines4-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines4-wf.cwl index 4c73d480c..8aada50e3 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines4-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines4-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: diff --git a/cwltool/schemas/draft-4/draft-4/count-lines5-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines5-wf.cwl index 2af0f8597..eb38e2231 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines5-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines5-wf.cwl @@ -1,11 +1,11 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: type: File - default: {class: File, path: hello.txt} + default: {class: File, location: hello.txt} outputs: count_output: type: int diff --git a/cwltool/schemas/draft-4/draft-4/count-lines6-job.json b/cwltool/schemas/draft-4/draft-4/count-lines6-job.json index 4db2d1d70..3652ded9d 100644 --- a/cwltool/schemas/draft-4/draft-4/count-lines6-job.json +++ b/cwltool/schemas/draft-4/draft-4/count-lines6-job.json @@ -2,21 +2,21 @@ "file1": [ { "class": "File", - "path": "whale.txt" + "location": "whale.txt" }, { "class": "File", - "path": "whale.txt" + "location": "whale.txt" } ], "file2": [ { "class": "File", - "path": "hello.txt" + "location": "hello.txt" }, { "class": "File", - "path": "hello.txt" + "location": "hello.txt" } ] } diff --git a/cwltool/schemas/draft-4/draft-4/count-lines6-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines6-wf.cwl index 69accb422..36296bffa 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines6-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines6-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: File[] diff --git a/cwltool/schemas/draft-4/draft-4/count-lines7-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines7-wf.cwl index 1394fcf1e..cf0e3d8dd 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines7-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines7-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: MultipleInputFeatureRequirement diff --git a/cwltool/schemas/draft-4/draft-4/count-lines8-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines8-wf.cwl index f743718fb..02895b549 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines8-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines8-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: File diff --git a/cwltool/schemas/draft-4/draft-4/count-lines9-wf.cwl b/cwltool/schemas/draft-4/draft-4/count-lines9-wf.cwl index aff39fcb8..9a340adc6 100755 --- a/cwltool/schemas/draft-4/draft-4/count-lines9-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/count-lines9-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: [] @@ -16,7 +16,7 @@ steps: file1: default: class: File - path: whale.txt + location: whale.txt out: [output] step2: diff --git a/cwltool/schemas/draft-4/draft-4/dir-job.yml b/cwltool/schemas/draft-4/draft-4/dir-job.yml new file mode 100644 index 000000000..30392cfc0 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir-job.yml @@ -0,0 +1,3 @@ +indir: + class: Directory + location: testdir \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/dir.cwl b/cwltool/schemas/draft-4/draft-4/dir.cwl new file mode 100644 index 000000000..0cfe978e8 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +requirements: + - class: ShellCommandRequirement +inputs: + indir: Directory +outputs: + outlist: + type: File + outputBinding: + glob: output.txt +baseCommand: [] +arguments: ["cd", "$(inputs.indir.path)", + {shellQuote: false, valueFrom: "&&"}, + "find", ".", + {shellQuote: false, valueFrom: "|"}, + "sort"] +stdout: output.txt \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/dir2.cwl b/cwltool/schemas/draft-4/draft-4/dir2.cwl new file mode 100644 index 000000000..e64af2c7b --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir2.cwl @@ -0,0 +1,20 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +hints: + DockerRequirement: + dockerPull: debian:8 + ShellCommandRequirement: {} +inputs: + indir: Directory +outputs: + outlist: + type: File + outputBinding: + glob: output.txt +baseCommand: [] +arguments: ["cd", "$(inputs.indir.path)", + {shellQuote: false, valueFrom: "&&"}, + "find", ".", + {shellQuote: false, valueFrom: "|"}, + "sort"] +stdout: output.txt \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/dir3-job.yml b/cwltool/schemas/draft-4/draft-4/dir3-job.yml new file mode 100644 index 000000000..aff0e8036 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir3-job.yml @@ -0,0 +1,3 @@ +inf: + class: File + location: hello.tar \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/dir3.cwl b/cwltool/schemas/draft-4/draft-4/dir3.cwl new file mode 100644 index 000000000..61fb96eba --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir3.cwl @@ -0,0 +1,13 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +baseCommand: [tar, xvf] +inputs: + inf: + type: File + inputBinding: + position: 1 +outputs: + outdir: + type: Directory + outputBinding: + glob: . diff --git a/cwltool/schemas/draft-4/draft-4/dir4-job.yml b/cwltool/schemas/draft-4/draft-4/dir4-job.yml new file mode 100644 index 000000000..8da860d27 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir4-job.yml @@ -0,0 +1,10 @@ +inf: + class: File + location: hello.tar + secondaryFiles: + - class: File + location: index.py + - entryname: xtestdir + entry: + class: Directory + location: testdir diff --git a/cwltool/schemas/draft-4/draft-4/dir4.cwl b/cwltool/schemas/draft-4/draft-4/dir4.cwl new file mode 100644 index 000000000..07341cf7d --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir4.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +requirements: + - class: ShellCommandRequirement +inputs: + inf: File +outputs: + outlist: + type: File + outputBinding: + glob: output.txt +baseCommand: [] +arguments: ["cd", "$(inputs.inf.dirname)", + {shellQuote: false, valueFrom: "&&"}, + "find", ".", + {shellQuote: false, valueFrom: "|"}, + "sort"] +stdout: output.txt \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/dir5.cwl b/cwltool/schemas/draft-4/draft-4/dir5.cwl new file mode 100644 index 000000000..27d9ac9e9 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/dir5.cwl @@ -0,0 +1,18 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +requirements: + - class: ShellCommandRequirement + - class: InitialWorkDirRequirement + listing: $(inputs.indir.listing) +inputs: + indir: Directory +outputs: + outlist: + type: File + outputBinding: + glob: output.txt +baseCommand: [] +arguments: ["find", ".", + {shellQuote: false, valueFrom: "|"}, + "sort"] +stdout: output.txt \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/echo-tool.cwl b/cwltool/schemas/draft-4/draft-4/echo-tool.cwl index 85c113939..f207942ea 100644 --- a/cwltool/schemas/draft-4/draft-4/echo-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/echo-tool.cwl @@ -1,7 +1,7 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: in: type: Any diff --git a/cwltool/schemas/draft-4/draft-4/egrep-stderr-mediumcut.cwl b/cwltool/schemas/draft-4/draft-4/egrep-stderr-mediumcut.cwl index f780c7319..a224a5c50 100644 --- a/cwltool/schemas/draft-4/draft-4/egrep-stderr-mediumcut.cwl +++ b/cwltool/schemas/draft-4/draft-4/egrep-stderr-mediumcut.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Test of capturing stderr output in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/egrep-stderr-shortcut.cwl b/cwltool/schemas/draft-4/draft-4/egrep-stderr-shortcut.cwl index 9a70aa1d5..9a8c26aa1 100644 --- a/cwltool/schemas/draft-4/draft-4/egrep-stderr-shortcut.cwl +++ b/cwltool/schemas/draft-4/draft-4/egrep-stderr-shortcut.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Test of capturing stderr output in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/egrep-stderr.cwl b/cwltool/schemas/draft-4/draft-4/egrep-stderr.cwl index 4bba74962..86ec03d05 100644 --- a/cwltool/schemas/draft-4/draft-4/egrep-stderr.cwl +++ b/cwltool/schemas/draft-4/draft-4/egrep-stderr.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Test of capturing stderr output in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/env-tool1.cwl b/cwltool/schemas/draft-4/draft-4/env-tool1.cwl index 64f406392..af33b3b54 100644 --- a/cwltool/schemas/draft-4/draft-4/env-tool1.cwl +++ b/cwltool/schemas/draft-4/draft-4/env-tool1.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: in: string outputs: diff --git a/cwltool/schemas/draft-4/draft-4/env-tool2.cwl b/cwltool/schemas/draft-4/draft-4/env-tool2.cwl index db5977a8e..8df0b64da 100644 --- a/cwltool/schemas/draft-4/draft-4/env-tool2.cwl +++ b/cwltool/schemas/draft-4/draft-4/env-tool2.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: in: string outputs: diff --git a/cwltool/schemas/draft-4/draft-4/env-wf1.cwl b/cwltool/schemas/draft-4/draft-4/env-wf1.cwl index 11583218f..09268c5a1 100644 --- a/cwltool/schemas/draft-4/draft-4/env-wf1.cwl +++ b/cwltool/schemas/draft-4/draft-4/env-wf1.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: in: string diff --git a/cwltool/schemas/draft-4/draft-4/env-wf2.cwl b/cwltool/schemas/draft-4/draft-4/env-wf2.cwl index fd5b321e7..9de645356 100644 --- a/cwltool/schemas/draft-4/draft-4/env-wf2.cwl +++ b/cwltool/schemas/draft-4/draft-4/env-wf2.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: in: string diff --git a/cwltool/schemas/draft-4/draft-4/formattest-job.json b/cwltool/schemas/draft-4/draft-4/formattest-job.json index da49d0103..0ff024096 100644 --- a/cwltool/schemas/draft-4/draft-4/formattest-job.json +++ b/cwltool/schemas/draft-4/draft-4/formattest-job.json @@ -1,7 +1,7 @@ { "input": { "class": "File", - "path": "whale.txt", + "location": "whale.txt", "format": "edam:format_2330" } } diff --git a/cwltool/schemas/draft-4/draft-4/formattest.cwl b/cwltool/schemas/draft-4/draft-4/formattest.cwl index 80dff90e4..d97e9be4d 100644 --- a/cwltool/schemas/draft-4/draft-4/formattest.cwl +++ b/cwltool/schemas/draft-4/draft-4/formattest.cwl @@ -1,6 +1,6 @@ $namespaces: edam: "http://edamontology.org/" -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool description: "Reverse each line using the `rev` command" inputs: diff --git a/cwltool/schemas/draft-4/draft-4/formattest2-job.json b/cwltool/schemas/draft-4/draft-4/formattest2-job.json index c70f7fb49..f706f6ed1 100644 --- a/cwltool/schemas/draft-4/draft-4/formattest2-job.json +++ b/cwltool/schemas/draft-4/draft-4/formattest2-job.json @@ -1,7 +1,7 @@ { "input": { "class": "File", - "path": "ref.fasta", + "location": "ref.fasta", "format": "edam:format_1929" } } diff --git a/cwltool/schemas/draft-4/draft-4/formattest2.cwl b/cwltool/schemas/draft-4/draft-4/formattest2.cwl index 377198de8..7ed3f9daa 100644 --- a/cwltool/schemas/draft-4/draft-4/formattest2.cwl +++ b/cwltool/schemas/draft-4/draft-4/formattest2.cwl @@ -3,7 +3,7 @@ $namespaces: $schemas: - EDAM.owl class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Reverse each line using the `rev` command" inputs: diff --git a/cwltool/schemas/draft-4/draft-4/formattest3.cwl b/cwltool/schemas/draft-4/draft-4/formattest3.cwl index fa4237699..e6342dfb4 100644 --- a/cwltool/schemas/draft-4/draft-4/formattest3.cwl +++ b/cwltool/schemas/draft-4/draft-4/formattest3.cwl @@ -5,7 +5,7 @@ $schemas: - EDAM.owl - gx_edam.ttl class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Reverse each line using the `rev` command" inputs: diff --git a/cwltool/schemas/draft-4/draft-4/glob-expr-list.cwl b/cwltool/schemas/draft-4/draft-4/glob-expr-list.cwl index 984142779..6144a785c 100644 --- a/cwltool/schemas/draft-4/draft-4/glob-expr-list.cwl +++ b/cwltool/schemas/draft-4/draft-4/glob-expr-list.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: ids: diff --git a/cwltool/schemas/draft-4/draft-4/metadata.cwl b/cwltool/schemas/draft-4/draft-4/metadata.cwl index b49e3f447..71c55e61a 100644 --- a/cwltool/schemas/draft-4/draft-4/metadata.cwl +++ b/cwltool/schemas/draft-4/draft-4/metadata.cwl @@ -6,7 +6,7 @@ $schemas: - foaf.rdf - dcterms.rdf -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool description: "Print the contents of a file to stdout using 'cat' running in a docker container." diff --git a/cwltool/schemas/draft-4/draft-4/null-expression1-tool.cwl b/cwltool/schemas/draft-4/draft-4/null-expression1-tool.cwl index 3460452f4..d953be546 100644 --- a/cwltool/schemas/draft-4/draft-4/null-expression1-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/null-expression1-tool.cwl @@ -3,7 +3,7 @@ class: ExpressionTool requirements: - class: InlineJavascriptRequirement -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: i1: diff --git a/cwltool/schemas/draft-4/draft-4/null-expression2-tool.cwl b/cwltool/schemas/draft-4/draft-4/null-expression2-tool.cwl index 129e26968..8cfb02dc8 100644 --- a/cwltool/schemas/draft-4/draft-4/null-expression2-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/null-expression2-tool.cwl @@ -3,7 +3,7 @@ class: ExpressionTool requirements: - class: InlineJavascriptRequirement -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: i1: Any diff --git a/cwltool/schemas/draft-4/draft-4/optional-output.cwl b/cwltool/schemas/draft-4/draft-4/optional-output.cwl index 46be1cdfe..ea058d3e9 100644 --- a/cwltool/schemas/draft-4/draft-4/optional-output.cwl +++ b/cwltool/schemas/draft-4/draft-4/optional-output.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: "cwl:draft-4.dev2" +cwlVersion: "cwl:draft-4.dev3" description: "Print the contents of a file to stdout using 'cat' running in a docker container." hints: DockerRequirement: diff --git a/cwltool/schemas/draft-4/draft-4/params.cwl b/cwltool/schemas/draft-4/draft-4/params.cwl index 774023f48..6fd43a50f 100644 --- a/cwltool/schemas/draft-4/draft-4/params.cwl +++ b/cwltool/schemas/draft-4/draft-4/params.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: bar: type: Any diff --git a/cwltool/schemas/draft-4/draft-4/params2.cwl b/cwltool/schemas/draft-4/draft-4/params2.cwl index 9993cef36..01967fcb3 100644 --- a/cwltool/schemas/draft-4/draft-4/params2.cwl +++ b/cwltool/schemas/draft-4/draft-4/params2.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/parseInt-job.json b/cwltool/schemas/draft-4/draft-4/parseInt-job.json index db0c8f496..b584ea23b 100644 --- a/cwltool/schemas/draft-4/draft-4/parseInt-job.json +++ b/cwltool/schemas/draft-4/draft-4/parseInt-job.json @@ -1,6 +1,6 @@ { "file1": { "class": "File", - "path": "number.txt" + "location": "number.txt" } } diff --git a/cwltool/schemas/draft-4/draft-4/parseInt-tool.cwl b/cwltool/schemas/draft-4/draft-4/parseInt-tool.cwl index bf1cc5a2d..5c3a77557 100755 --- a/cwltool/schemas/draft-4/draft-4/parseInt-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/parseInt-tool.cwl @@ -3,7 +3,7 @@ class: ExpressionTool requirements: - class: InlineJavascriptRequirement -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: diff --git a/cwltool/schemas/draft-4/draft-4/record-output-job.json b/cwltool/schemas/draft-4/draft-4/record-output-job.json index 44d008189..df29550c2 100644 --- a/cwltool/schemas/draft-4/draft-4/record-output-job.json +++ b/cwltool/schemas/draft-4/draft-4/record-output-job.json @@ -1,6 +1,6 @@ { "irec": { - "ifoo": {"path": "whale.txt", "class": "File"}, - "ibar": {"path": "ref.fasta", "class": "File"} + "ifoo": {"location": "whale.txt", "class": "File"}, + "ibar": {"location": "ref.fasta", "class": "File"} } } \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/record-output.cwl b/cwltool/schemas/draft-4/draft-4/record-output.cwl index b85251464..2e718da98 100644 --- a/cwltool/schemas/draft-4/draft-4/record-output.cwl +++ b/cwltool/schemas/draft-4/draft-4/record-output.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: ShellCommandRequirement inputs: diff --git a/cwltool/schemas/draft-4/draft-4/rename-job.json b/cwltool/schemas/draft-4/draft-4/rename-job.json index 917bc989d..c8ff96069 100644 --- a/cwltool/schemas/draft-4/draft-4/rename-job.json +++ b/cwltool/schemas/draft-4/draft-4/rename-job.json @@ -1,6 +1,6 @@ { "srcfile": { - "path": "whale.txt", + "location": "whale.txt", "class": "File" }, "newname": "fish.txt" diff --git a/cwltool/schemas/draft-4/draft-4/rename.cwl b/cwltool/schemas/draft-4/draft-4/rename.cwl index 7b9de2931..e97743ee0 100644 --- a/cwltool/schemas/draft-4/draft-4/rename.cwl +++ b/cwltool/schemas/draft-4/draft-4/rename.cwl @@ -1,11 +1,11 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 baseCommand: "true" requirements: - CreateFileRequirement: - fileDef: - - filename: $(inputs.newname) - fileContent: $(inputs.srcfile) + InitialWorkDirRequirement: + listing: + - entryname: $(inputs.newname) + entry: $(inputs.srcfile) inputs: srcfile: File newname: string diff --git a/cwltool/schemas/draft-4/draft-4/revsort-job.json b/cwltool/schemas/draft-4/draft-4/revsort-job.json index 3c6d0f889..f5671aab2 100644 --- a/cwltool/schemas/draft-4/draft-4/revsort-job.json +++ b/cwltool/schemas/draft-4/draft-4/revsort-job.json @@ -1,6 +1,6 @@ { "input": { "class": "File", - "path": "whale.txt" + "location": "whale.txt" } } diff --git a/cwltool/schemas/draft-4/draft-4/revsort.cwl b/cwltool/schemas/draft-4/draft-4/revsort.cwl index 8ca76ee08..e94b30ada 100644 --- a/cwltool/schemas/draft-4/draft-4/revsort.cwl +++ b/cwltool/schemas/draft-4/draft-4/revsort.cwl @@ -3,7 +3,7 @@ # class: Workflow description: "Reverse the lines in a document, then sort those lines." -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 # Requirements & hints specify prerequisites and extensions to the workflow. # In this example, DockerRequirement specifies a default Docker container diff --git a/cwltool/schemas/draft-4/draft-4/revtool.cwl b/cwltool/schemas/draft-4/draft-4/revtool.cwl index 2a276f57d..ba3033c09 100644 --- a/cwltool/schemas/draft-4/draft-4/revtool.cwl +++ b/cwltool/schemas/draft-4/draft-4/revtool.cwl @@ -2,7 +2,7 @@ # Simplest example command line program wrapper for the Unix tool "rev". # class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Reverse each line using the `rev` command" # The "inputs" array defines the structure of the input object that describes diff --git a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf1.cwl b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf1.cwl index ab36fb4fa..0256066b3 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf1.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf1.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: Workflow inputs: inp: diff --git a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf2.cwl b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf2.cwl index f59ed9618..1894abee6 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf2.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf2.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: Workflow inputs: diff --git a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf3.cwl b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf3.cwl index 0e9192d2f..a691acf5d 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf3.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf3.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: echo diff --git a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf4.cwl b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf4.cwl index aaf763da4..9cae1a2b2 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf4.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf4.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: echo class: CommandLineTool diff --git a/cwltool/schemas/draft-4/draft-4/scatter-wf1.cwl b/cwltool/schemas/draft-4/draft-4/scatter-wf1.cwl index eec267915..a8ffc6531 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-wf1.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-wf1.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: Workflow inputs: inp: string[] diff --git a/cwltool/schemas/draft-4/draft-4/scatter-wf2.cwl b/cwltool/schemas/draft-4/draft-4/scatter-wf2.cwl index adb798a5c..abc20be26 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-wf2.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-wf2.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: Workflow inputs: diff --git a/cwltool/schemas/draft-4/draft-4/scatter-wf3.cwl b/cwltool/schemas/draft-4/draft-4/scatter-wf3.cwl index 7cfe1250c..ce32fbd69 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-wf3.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-wf3.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: echo diff --git a/cwltool/schemas/draft-4/draft-4/scatter-wf4.cwl b/cwltool/schemas/draft-4/draft-4/scatter-wf4.cwl index 8ff37bc53..c01d5650d 100644 --- a/cwltool/schemas/draft-4/draft-4/scatter-wf4.cwl +++ b/cwltool/schemas/draft-4/draft-4/scatter-wf4.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: echo class: CommandLineTool diff --git a/cwltool/schemas/draft-4/draft-4/schemadef-tool.cwl b/cwltool/schemas/draft-4/draft-4/schemadef-tool.cwl index 33e6f20f3..0d8447442 100644 --- a/cwltool/schemas/draft-4/draft-4/schemadef-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/schemadef-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - $import: schemadef-type.yml - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/schemadef-wf.cwl b/cwltool/schemas/draft-4/draft-4/schemadef-wf.cwl index 15b873e80..4667f7894 100644 --- a/cwltool/schemas/draft-4/draft-4/schemadef-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/schemadef-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: Workflow requirements: diff --git a/cwltool/schemas/draft-4/draft-4/search-job.json b/cwltool/schemas/draft-4/draft-4/search-job.json index 0a6735da8..b5a1b61e5 100644 --- a/cwltool/schemas/draft-4/draft-4/search-job.json +++ b/cwltool/schemas/draft-4/draft-4/search-job.json @@ -1,7 +1,7 @@ { "infile": { "class": "File", - "path": "whale.txt" + "location": "whale.txt" }, "term": "find" } diff --git a/cwltool/schemas/draft-4/draft-4/search.cwl b/cwltool/schemas/draft-4/draft-4/search.cwl index 104a26437..dc32c0955 100644 --- a/cwltool/schemas/draft-4/draft-4/search.cwl +++ b/cwltool/schemas/draft-4/draft-4/search.cwl @@ -1,4 +1,4 @@ -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 $graph: - id: index class: CommandLineTool @@ -7,10 +7,9 @@ $graph: - valueFrom: input.txt position: 1 requirements: - - class: CreateFileRequirement - fileDef: - - filename: input.txt - fileContent: $(inputs.file) + - class: InitialWorkDirRequirement + listing: + input.txt: $(inputs.file) - class: InlineJavascriptRequirement inputs: @@ -19,7 +18,7 @@ $graph: type: File default: class: File - path: index.py + location: index.py inputBinding: position: 0 outputs: @@ -30,9 +29,9 @@ $graph: secondaryFiles: - ".idx1" - "^.idx2" - - '$(self.path+".idx3")' - - '$({"path": self.path+".idx4", "class": "File"})' - - '${ return self.path+".idx5"; }' + - '$(self.location+".idx3")' + - '$({"location": self.location+".idx4", "class": "File"})' + - '${ return self.location+".idx5"; }' - id: search class: CommandLineTool @@ -47,14 +46,14 @@ $graph: secondaryFiles: - ".idx1" - "^.idx2" - - '$(self.path+".idx3")' - - '$({"path": self.path+".idx4", "class": "File"})' - - '${ return self.path+".idx5"; }' + - '$(self.location+".idx3")' + - '$({"location": self.location+".idx4", "class": "File"})' + - '${ return self.location+".idx5"; }' search.py: type: File default: class: File - path: search.py + location: search.py inputBinding: position: 0 term: diff --git a/cwltool/schemas/draft-4/draft-4/shelltest.cwl b/cwltool/schemas/draft-4/draft-4/shelltest.cwl index 0ae27a3cb..9ccb34d52 100644 --- a/cwltool/schemas/draft-4/draft-4/shelltest.cwl +++ b/cwltool/schemas/draft-4/draft-4/shelltest.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 description: "Reverse each line using the `rev` command then sort." requirements: - class: ShellCommandRequirement diff --git a/cwltool/schemas/draft-4/draft-4/sorttool.cwl b/cwltool/schemas/draft-4/draft-4/sorttool.cwl index 447a502d2..7aeeadc03 100644 --- a/cwltool/schemas/draft-4/draft-4/sorttool.cwl +++ b/cwltool/schemas/draft-4/draft-4/sorttool.cwl @@ -2,7 +2,7 @@ # demonstrating command line flags. class: CommandLineTool description: "Sort lines using the `sort` command" -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 # This example is similar to the previous one, with an additional input # parameter called "reverse". It is a boolean parameter, which is diff --git a/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.cwl b/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.cwl index ba8418131..75b251d02 100644 --- a/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: StepInputExpressionRequirement diff --git a/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.json b/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.json index 8d6a9a6f1..b13324174 100644 --- a/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.json +++ b/cwltool/schemas/draft-4/draft-4/step-valuefrom-wf.json @@ -2,7 +2,7 @@ "in": { "file1": { "class": "File", - "path": "whale.txt" + "location": "whale.txt" } } } diff --git a/cwltool/schemas/draft-4/draft-4/step-valuefrom2-wf.cwl b/cwltool/schemas/draft-4/draft-4/step-valuefrom2-wf.cwl index c5c9b8160..8d085aa37 100644 --- a/cwltool/schemas/draft-4/draft-4/step-valuefrom2-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/step-valuefrom2-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: StepInputExpressionRequirement - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/step-valuefrom3-wf.cwl b/cwltool/schemas/draft-4/draft-4/step-valuefrom3-wf.cwl index bd1acc8a8..54f70ba85 100644 --- a/cwltool/schemas/draft-4/draft-4/step-valuefrom3-wf.cwl +++ b/cwltool/schemas/draft-4/draft-4/step-valuefrom3-wf.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: Workflow -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: StepInputExpressionRequirement - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/template-tool.cwl b/cwltool/schemas/draft-4/draft-4/template-tool.cwl index 7e3521382..f7a00024f 100755 --- a/cwltool/schemas/draft-4/draft-4/template-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/template-tool.cwl @@ -1,5 +1,5 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool requirements: - class: DockerRequirement @@ -8,10 +8,9 @@ requirements: expressionLib: - { $include: underscore.js } - "var t = function(s) { return _.template(s)({'inputs': inputs}); };" - - class: CreateFileRequirement - fileDef: - - filename: foo.txt - fileContent: > + - class: InitialWorkDirRequirement + listing: + foo.txt: > $(t("The file is <%= inputs.file1.path.split('/').slice(-1)[0] %>\n")) inputs: - id: file1 diff --git a/cwltool/schemas/draft-4/draft-4/test-cwl-out.cwl b/cwltool/schemas/draft-4/draft-4/test-cwl-out.cwl index 5ce6331be..884054d74 100644 --- a/cwltool/schemas/draft-4/draft-4/test-cwl-out.cwl +++ b/cwltool/schemas/draft-4/draft-4/test-cwl-out.cwl @@ -1,5 +1,5 @@ class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: ShellCommandRequirement - class: DockerRequirement diff --git a/cwltool/schemas/draft-4/draft-4/testdir/a b/cwltool/schemas/draft-4/draft-4/testdir/a new file mode 100644 index 000000000..e69de29bb diff --git a/cwltool/schemas/draft-4/draft-4/testdir/b b/cwltool/schemas/draft-4/draft-4/testdir/b new file mode 100644 index 000000000..e69de29bb diff --git a/cwltool/schemas/draft-4/draft-4/testdir/c/d b/cwltool/schemas/draft-4/draft-4/testdir/c/d new file mode 100644 index 000000000..e69de29bb diff --git a/cwltool/schemas/draft-4/draft-4/tmap-job.json b/cwltool/schemas/draft-4/draft-4/tmap-job.json index b4fec8a09..24cdda68a 100644 --- a/cwltool/schemas/draft-4/draft-4/tmap-job.json +++ b/cwltool/schemas/draft-4/draft-4/tmap-job.json @@ -1,7 +1,7 @@ { "reads": { "class": "File", - "path": "reads.fastq" + "location": "reads.fastq" }, "stages": [ { diff --git a/cwltool/schemas/draft-4/draft-4/tmap-tool.cwl b/cwltool/schemas/draft-4/draft-4/tmap-tool.cwl index 8f8aca16a..1b6273e26 100755 --- a/cwltool/schemas/draft-4/draft-4/tmap-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/tmap-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner { - "cwlVersion": "cwl:draft-4.dev2", + "cwlVersion": "cwl:draft-4.dev3", "class": "CommandLineTool", @@ -24,7 +24,7 @@ type: File, default: { class: File, - path: args.py + location: args.py }, inputBinding: { position: -1 diff --git a/cwltool/schemas/draft-4/draft-4/wc-job.json b/cwltool/schemas/draft-4/draft-4/wc-job.json index 32ffc842b..598568d38 100644 --- a/cwltool/schemas/draft-4/draft-4/wc-job.json +++ b/cwltool/schemas/draft-4/draft-4/wc-job.json @@ -1,6 +1,6 @@ { "file1": { "class": "File", - "path": "whale.txt" + "location": "whale.txt" } } diff --git a/cwltool/schemas/draft-4/draft-4/wc-tool.cwl b/cwltool/schemas/draft-4/draft-4/wc-tool.cwl index f129aaf0d..08ef2e77b 100755 --- a/cwltool/schemas/draft-4/draft-4/wc-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/wc-tool.cwl @@ -1,7 +1,7 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 inputs: file1: File diff --git a/cwltool/schemas/draft-4/draft-4/wc2-tool.cwl b/cwltool/schemas/draft-4/draft-4/wc2-tool.cwl index 4e5704e7d..510a6dc5c 100755 --- a/cwltool/schemas/draft-4/draft-4/wc2-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/wc2-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/wc3-tool.cwl b/cwltool/schemas/draft-4/draft-4/wc3-tool.cwl index f7fff7e5a..6aa6986f4 100644 --- a/cwltool/schemas/draft-4/draft-4/wc3-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/wc3-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: InlineJavascriptRequirement diff --git a/cwltool/schemas/draft-4/draft-4/wc4-tool.cwl b/cwltool/schemas/draft-4/draft-4/wc4-tool.cwl index 4a4404579..21e08a530 100644 --- a/cwltool/schemas/draft-4/draft-4/wc4-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/wc4-tool.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 requirements: - class: InlineJavascriptRequirement diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 618d35ac0..cfdcb657b 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -13,10 +13,19 @@ def _abs(self, p): # type: (unicode) -> unicode return abspath(p, self.basedir) def glob(self, pattern): # type: (unicode) -> List[unicode] - return glob.glob(self._abs(pattern)) + return ["file://%s" % self._abs(l) for l in glob.glob(self._abs(pattern))] def open(self, fn, mode): # type: (unicode, str) -> BinaryIO return open(self._abs(fn), mode) def exists(self, fn): # type: (unicode) -> bool return os.path.exists(self._abs(fn)) + + def isfile(self, fn): + return os.path.isfile(self._abs(fn)) + + def isdir(self, fn): + return os.path.isdir(self._abs(fn)) + + def listdir(self, fn): + return [abspath(l, fn) for l in os.listdir(self._abs(fn))] diff --git a/cwltool/update.py b/cwltool/update.py index 19a70a56b..dfe76b14e 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -324,11 +324,27 @@ def _draft3toDraft4dev1(doc, loader, baseuri): # type: (Any, Loader, str) -> Any if isinstance(doc, dict): if "class" in doc and doc["class"] == "Workflow": + def fixup(f): + doc, frg = urlparse.urldefrag(f) + frg = '/'.join(frg.rsplit('.', 1)) + return doc + "#" + frg + for step in doc["steps"]: step["in"] = step["inputs"] step["out"] = step["outputs"] del step["inputs"] del step["outputs"] + for io in ("in", "out"): + for i in step[io]: + i["id"] = fixup(i["id"]) + if "source" in i: + i["source"] = [fixup(s) for s in aslist(i["source"])] + if len(i["source"]) == 1: + i["source"] = i["source"][0] + if "scatter" in step: + step["scatter"] = [fixup(s) for s in aslist(step["scatter"])] + for out in doc["outputs"]: + out["source"] = fixup(out["source"]) for key, value in doc.items(): doc[key] = _draft3toDraft4dev1(value, loader, baseuri) elif isinstance(doc, list): @@ -360,6 +376,39 @@ def draft4Dev1toDev2(doc, loader, baseuri): """Public updater for draft-4.dev1 to draft-4.dev2.""" return (_draft4Dev1toDev2(doc, loader, baseuri), "draft-4.dev2") + +def _draft4Dev2toDev3(doc, loader, baseuri): + # type: (Any, Loader, str) -> Any + if isinstance(doc, dict): + if "class" in doc and doc["class"] == "File": + doc["location"] = doc["path"] + del doc["path"] + if "secondaryFiles" in doc: + for i, sf in enumerate(doc["secondaryFiles"]): + if "$(" in sf or "${" in sf: + doc["secondaryFiles"][i] = sf.replace('"path"', '"location"').replace(".path", ".location") + + if "class" in doc and doc["class"] == "CreateFileRequirement": + doc["class"] = "InitialWorkDirRequirement" + doc["listing"] = [] + for f in doc["fileDef"]: + doc["listing"].append({ + "entryname": f["filename"], + "entry": f["fileContent"] + }) + del doc["fileDef"] + for key, value in doc.items(): + doc[key] = _draft4Dev2toDev3(value, loader, baseuri) + elif isinstance(doc, list): + doc = [_draft4Dev2toDev3(item, loader, baseuri) for item in doc] + + return doc + +def draft4Dev2toDev3(doc, loader, baseuri): + # type: (Any, Loader, str) -> Tuple[Any, str] + """Public updater for draft-4.dev2 to draft-4.dev3.""" + return (_draft4Dev2toDev3(doc, loader, baseuri), "draft-4.dev3") + UPDATES = { "draft-2": draft2toDraft3dev1, "draft-3": draft3toDraft4dev1 @@ -372,13 +421,14 @@ def draft4Dev1toDev2(doc, loader, baseuri): "draft-3.dev4": draftDraft3dev4toDev5, "draft-3.dev5": draftDraft3dev5toFinal, "draft-4.dev1": draft4Dev1toDev2, - "draft-4.dev2": None + "draft-4.dev2": draft4Dev2toDev3, + "draft-4.dev3": None } # type: Dict[unicode, Callable[[Any, Loader, str], Tuple[Any, str]]] ALLUPDATES = UPDATES.copy() ALLUPDATES.update(DEVUPDATES) -LATEST = "draft-4.dev2" +LATEST = "draft-4.dev3" def identity(doc, loader, baseuri): # pylint: disable=unused-argument # type: (Any, Loader, str) -> Tuple[Any, Union[str, unicode]] diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 48c0069da..e793d901a 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -298,7 +298,7 @@ def valueFromFunc(k, v): # type: (Any, Any) -> Any def run(self, **kwargs): _logger.debug(u"[%s] workflow starting", self.name) - def job(self, joborder, output_callback, move_outputs=True, **kwargs): + def job(self, joborder, output_callback, **kwargs): # type: (Dict[unicode, Any], Callable[[Any, Any], Any], bool, **Any) -> Generator[WorkflowJob, None, None] self.state = {} self.processStatus = "success" @@ -319,8 +319,6 @@ def job(self, joborder, output_callback, move_outputs=True, **kwargs): for out in s.tool["outputs"]: self.state[out["id"]] = None - output_dirs = set() - completed = 0 while completed < len(self.steps) and self.processStatus == "success": made_progress = False @@ -333,8 +331,6 @@ def job(self, joborder, output_callback, move_outputs=True, **kwargs): for newjob in step.iterable: if newjob: made_progress = True - if newjob.outdir: - output_dirs.add(newjob.outdir) yield newjob else: break @@ -351,43 +347,6 @@ def job(self, joborder, output_callback, move_outputs=True, **kwargs): if wo is None: raise WorkflowException("Output for workflow not available") - if move_outputs: - targets = set() # type: Set[str] - conflicts = set() - - outfiles = findfiles(wo) - - for f in outfiles: - for a in output_dirs: - if f["path"].startswith(a): - src = f["path"] - dst = os.path.join(self.outdir, src[len(a)+1:]) - if dst in targets: - conflicts.add(dst) - else: - targets.add(dst) - - for f in outfiles: - for a in output_dirs: - if f["path"].startswith(a): - src = f["path"] - dst = os.path.join(self.outdir, src[len(a)+1:]) - if dst in conflicts: - sp = os.path.splitext(dst) - dst = u"%s-%s%s" % (sp[0], str(random.randint(1, 1000000000)), sp[1]) - dirname = os.path.dirname(dst) - if not os.path.exists(dirname): - os.makedirs(dirname) - _logger.debug(u"[%s] Moving '%s' to '%s'", self.name, src, dst) - shutil.move(src, dst) - f["path"] = dst - - for a in output_dirs: - if os.path.exists(a) and empty_subtree(a): - if kwargs.get("rm_tmpdir", True): - _logger.debug(u"[%s] Removing intermediate output directory %s", self.name, a) - shutil.rmtree(a, True) - _logger.info(u"[%s] outdir is %s", self.name, self.outdir) output_callback(wo, self.processStatus) diff --git a/tests/echo.cwl b/tests/echo.cwl index 6eb6a23a6..da6943328 100644 --- a/tests/echo.cwl +++ b/tests/echo.cwl @@ -1,4 +1,4 @@ -cwlVersion: cwl:draft-3 +cwlVersion: cwl:draft-4.dev2 class: CommandLineTool inputs: - id: inp diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index 7e4af9e5b..b54c89a3f 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -9,13 +9,12 @@ class TestPathMapper(unittest.TestCase): def test_subclass(self): - + class SubPathMapper(PathMapper): - def __init__(self, referenced_files, basedir, new): - super(SubPathMapper, self).__init__(referenced_files, basedir) + def __init__(self, referenced_files, basedir, stagedir, new): + super(SubPathMapper, self).__init__(referenced_files, basedir, stagedir) self.new = new - a = SubPathMapper([], '', "new") + a = SubPathMapper([], '', '', "new") self.assertTrue(a.new, "new") - diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 6c3e05680..dfa108318 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -8,7 +8,7 @@ class ToolArgparse(unittest.TestCase): script=''' #!/usr/bin/env cwl-runner -cwlVersion: "draft-3" +cwlVersion: "draft-4.dev2" class: CommandLineTool description: "This tool is developed for SMC-RNA Challenge for detecting gene fusions (STAR fusion)" inputs: @@ -28,7 +28,7 @@ class ToolArgparse(unittest.TestCase): script2=''' #!/usr/bin/env cwl-runner -cwlVersion: 'cwl:draft-3' +cwlVersion: 'cwl:draft-4.dev2' class: CommandLineTool inputs: - id: bdg