From 9c1f0814a068cc3cdf10a2e2bfac911e92de6877 Mon Sep 17 00:00:00 2001 From: Geet Duggal Date: Tue, 12 Sep 2017 22:39:42 +0000 Subject: [PATCH 1/5] Support for user space docker --- cwltool/job.py | 82 ++++++++++++++++++++++++++++--------------------- cwltool/main.py | 2 ++ 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 3dd6ae71d..18bfd6ea6 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -360,32 +360,39 @@ def run(self, pull_image=True, rm_container=True, img_id = None env = None # type: MutableMapping[Text, Text] - try: - env = cast(MutableMapping[Text, Text], os.environ) - if docker_req and kwargs.get("use_container"): - img_id = docker.get_from_requirements(docker_req, True, pull_image) - if img_id is None: - if self.builder.find_default_container: - default_container = self.builder.find_default_container() - if default_container: - img_id = default_container - env = cast(MutableMapping[Text, Text], os.environ) - - if docker_req and img_id is None and kwargs.get("use_container"): - raise Exception("Docker image not available") - except Exception as e: - _logger.debug("Docker error", exc_info=True) - if docker_is_req: - raise UnsupportedRequirement( - "Docker is required to run this tool: %s" % e) - else: - raise WorkflowException( - "Docker is not available for this tool, try --no-container" - " to disable Docker: %s" % e) + user_space_docker_cmd = kwargs.get("user_space_docker_cmd") + if docker_req and user_space_docker_cmd: + img_id = str(docker_req["dockerPull"]) + else: + try: + env = cast(MutableMapping[Text, Text], os.environ) + if docker_req and kwargs.get("use_container"): + img_id = docker.get_from_requirements(docker_req, True, pull_image) + if img_id is None: + if self.builder.find_default_container: + default_container = self.builder.find_default_container() + if default_container: + img_id = default_container + env = cast(MutableMapping[Text, Text], os.environ) + + if docker_req and img_id is None and kwargs.get("use_container"): + raise Exception("Docker image not available") + except Exception as e: + _logger.debug("Docker error", exc_info=True) + if docker_is_req: + raise UnsupportedRequirement( + "Docker is required to run this tool: %s" % e) + else: + raise WorkflowException( + "Docker is not available for this tool, try --no-container" + " to disable Docker: %s" % e) self._setup(kwargs) - runtime = [u"docker", u"run", u"-i"] + if user_space_docker_cmd: + runtime = [user_space_docker_cmd, u"run"] + else: + runtime = [u"docker", u"run", u"-i"] runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir)) runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) @@ -394,23 +401,28 @@ def run(self, pull_image=True, rm_container=True, if self.generatemapper: self.add_volumes(self.generatemapper, runtime) + if user_space_docker_cmd: + runtime = [x.replace(":ro", "") for x in runtime] + runtime = [x.replace(":rw", "") for x in runtime] + runtime.append(u"--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir))) - runtime.append(u"--read-only=true") + if not user_space_docker_cmd: + runtime.append(u"--read-only=true") - if kwargs.get("custom_net", None) is not None: - runtime.append(u"--net={0}".format(kwargs.get("custom_net"))) - elif kwargs.get("disable_net", None): - runtime.append(u"--net=none") + if kwargs.get("custom_net", None) is not None: + runtime.append(u"--net={0}".format(kwargs.get("custom_net"))) + elif kwargs.get("disable_net", None): + runtime.append(u"--net=none") - if self.stdout: - runtime.append("--log-driver=none") + if self.stdout: + runtime.append("--log-driver=none") - euid, egid = docker_vm_id() - if not onWindows(): # MS Windows does not have getuid() or geteuid() functions - euid, egid = euid or os.geteuid(), egid or os.getgid() + euid, egid = docker_vm_id() + if not onWindows(): # MS Windows does not have getuid() or geteuid() functions + euid, egid = euid or os.geteuid(), egid or os.getgid() - if kwargs.get("no_match_user", None) is False and (euid, egid) != (None, None): - runtime.append(u"--user=%d:%d" % (euid, egid)) + if kwargs.get("no_match_user", None) is False and (euid, egid) != (None, None): + runtime.append(u"--user=%d:%d" % (euid, egid)) if rm_container: runtime.append(u"--rm") diff --git a/cwltool/main.py b/cwltool/main.py index 4330cd421..67d68aa67 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -159,6 +159,8 @@ def arg_parser(): # type: () -> argparse.ArgumentParser exgroup.add_argument("--debug", action="store_true", help="Print even more logging") parser.add_argument("--js-console", action="store_true", help="Enable javascript console output") + parser.add_argument("--user-space-docker-cmd", + help="Specify a user space docker command that will be used to call 'pull' and 'run'") dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS From d9acb537b1273d0db1d4487d755239d30391c9a1 Mon Sep 17 00:00:00 2001 From: Geet Duggal Date: Wed, 13 Sep 2017 17:46:25 +0000 Subject: [PATCH 2/5] Try to fix some str/unicode errors? --- cwltool/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 18bfd6ea6..4b6e5fb97 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -367,12 +367,12 @@ def run(self, pull_image=True, rm_container=True, try: env = cast(MutableMapping[Text, Text], os.environ) if docker_req and kwargs.get("use_container"): - img_id = docker.get_from_requirements(docker_req, True, pull_image) + img_id = str(docker.get_from_requirements(docker_req, True, pull_image)) if img_id is None: if self.builder.find_default_container: default_container = self.builder.find_default_container() if default_container: - img_id = default_container + img_id = str(default_container) env = cast(MutableMapping[Text, Text], os.environ) if docker_req and img_id is None and kwargs.get("use_container"): From 3cc8e00bebd502d21a2c665b60fb5ec9524b5ede Mon Sep 17 00:00:00 2001 From: Geet Duggal Date: Sat, 7 Oct 2017 16:17:46 +0000 Subject: [PATCH 3/5] Allow for dockerImageId --- cwltool/job.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cwltool/job.py b/cwltool/job.py index b3484b4a1..d12651bc9 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -362,7 +362,13 @@ def run(self, pull_image=True, rm_container=True, env = None # type: MutableMapping[Text, Text] user_space_docker_cmd = kwargs.get("user_space_docker_cmd") if docker_req and user_space_docker_cmd: - img_id = str(docker_req["dockerPull"]) + # For user-space docker implementations, a local image name or ID takes precedence over a network pull + if 'dockerImageId' in docker_req: + img_id = str(docker_req["dockerImageId"]) + elif 'dockerPull' in docker_req: + img_id = str(docker_req["dockerPull"]) + else: + raise Exception("Docker image must be specified as 'dockerImageId' or 'dockerPull' when using user space implementation of Docker") else: try: env = cast(MutableMapping[Text, Text], os.environ) From 6922e95b41b28a31b7700eac1e49ae06c353aa60 Mon Sep 17 00:00:00 2001 From: Geet Duggal Date: Fri, 13 Oct 2017 00:43:44 +0000 Subject: [PATCH 4/5] Initial draft of documentation --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index cc8444259..9082f9c01 100644 --- a/README.rst +++ b/README.rst @@ -500,3 +500,18 @@ logger_handler logging.Handler Handler object for logging. + +Running user-space implementations of Docker +-------------------------------------------- + +Some compute environments disallow user-space installation of Docker due to incompatiblities in libraries or to meet security requirements. The CWL reference supports using a user space implementation with the `--user-space-docker-cmd` option. + +Example using `dx-docker` (https://wiki.dnanexus.com/Developer-Tutorials/Using-Docker-Images): + +For use on Linux, install the DNAnexus toolkit (see https://wiki.dnanexus.com/Downloads for instructions). + +Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests: + +``` +cwltool --user-space-docker-cmd=dx-docker --outdir=/tmp/tmpidytmp v1.0/test-cwl-out2.cwl v1.0/empty.json +``` From 48da00096fe1dd6fc1381a87b45e949c1a8ce53f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 13 Oct 2017 16:27:07 +0200 Subject: [PATCH 5/5] mention --user-space-docker-cmd option, reflow code (#1) --- cwltool/job.py | 56 ++++++++++++++++++++++++++++++++++--------------- cwltool/main.py | 4 +++- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index d12651bc9..6e6e4a55d 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -321,15 +321,20 @@ def add_volumes(self, pathmapper, runtime): if not vol.staged: continue if vol.target.startswith(container_outdir+"/"): - host_outdir_tgt = os.path.join(host_outdir, vol.target[len(container_outdir)+1:]) + host_outdir_tgt = os.path.join( + host_outdir, vol.target[len(container_outdir)+1:]) else: host_outdir_tgt = None if vol.type in ("File", "Directory"): if not vol.resolved.startswith("_:"): - runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target))) + runtime.append(u"--volume=%s:%s:ro" % ( + docker_windows_path_adjust(vol.resolved), + docker_windows_path_adjust(vol.target))) elif vol.type == "WritableFile": if self.inplace_update: - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target))) + runtime.append(u"--volume=%s:%s:rw" % ( + docker_windows_path_adjust(vol.resolved), + docker_windows_path_adjust(vol.target))) else: shutil.copy(vol.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt) @@ -338,7 +343,9 @@ def add_volumes(self, pathmapper, runtime): os.makedirs(vol.target, 0o0755) else: if self.inplace_update: - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(vol.target))) + runtime.append(u"--volume=%s:%s:rw" % ( + docker_windows_path_adjust(vol.resolved), + docker_windows_path_adjust(vol.target))) else: shutil.copytree(vol.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt) @@ -350,7 +357,9 @@ def add_volumes(self, pathmapper, runtime): fd, createtmp = tempfile.mkstemp(dir=self.tmpdir) with os.fdopen(fd, "wb") as f: f.write(vol.resolved.encode("utf-8")) - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(createtmp), docker_windows_path_adjust(vol.target))) + runtime.append(u"--volume=%s:%s:rw" % ( + docker_windows_path_adjust(createtmp), + docker_windows_path_adjust(vol.target))) def run(self, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs="move", **kwargs): @@ -362,18 +371,22 @@ def run(self, pull_image=True, rm_container=True, env = None # type: MutableMapping[Text, Text] user_space_docker_cmd = kwargs.get("user_space_docker_cmd") if docker_req and user_space_docker_cmd: - # For user-space docker implementations, a local image name or ID takes precedence over a network pull + # For user-space docker implementations, a local image name or ID + # takes precedence over a network pull if 'dockerImageId' in docker_req: img_id = str(docker_req["dockerImageId"]) elif 'dockerPull' in docker_req: img_id = str(docker_req["dockerPull"]) else: - raise Exception("Docker image must be specified as 'dockerImageId' or 'dockerPull' when using user space implementation of Docker") + raise Exception("Docker image must be specified as " + "'dockerImageId' or 'dockerPull' when using user " + "space implementations of Docker") else: try: env = cast(MutableMapping[Text, Text], os.environ) if docker_req and kwargs.get("use_container"): - img_id = str(docker.get_from_requirements(docker_req, True, pull_image)) + img_id = str(docker.get_from_requirements( + docker_req, True, pull_image)) if img_id is None: if self.builder.find_default_container: default_container = self.builder.find_default_container() @@ -390,8 +403,10 @@ def run(self, pull_image=True, rm_container=True, "Docker is required to run this tool: %s" % e) else: raise WorkflowException( - "Docker is not available for this tool, try --no-container" - " to disable Docker: %s" % e) + "Docker is not available for this tool, try " + "--no-container to disable Docker, or install " + "a user space Docker replacement like uDocker with " + "--user-space-docker-cmd.: %s" % e) self._setup(kwargs) @@ -400,8 +415,11 @@ def run(self, pull_image=True, rm_container=True, else: runtime = [u"docker", u"run", u"-i"] - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir)) - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) + runtime.append(u"--volume=%s:%s:rw" % ( + docker_windows_path_adjust(os.path.realpath(self.outdir)), + self.builder.outdir)) + runtime.append(u"--volume=%s:%s:rw" % ( + docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) self.add_volumes(self.pathmapper, runtime) if self.generatemapper: @@ -411,11 +429,12 @@ def run(self, pull_image=True, rm_container=True, runtime = [x.replace(":ro", "") for x in runtime] runtime = [x.replace(":rw", "") for x in runtime] - runtime.append(u"--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir))) + runtime.append(u"--workdir=%s" % ( + docker_windows_path_adjust(self.builder.outdir))) if not user_space_docker_cmd: if not kwargs.get("no_read_only"): - runtime.append(u"--read-only=true") + runtime.append(u"--read-only=true") if kwargs.get("custom_net", None) is not None: runtime.append(u"--net={0}".format(kwargs.get("custom_net"))) @@ -426,10 +445,12 @@ def run(self, pull_image=True, rm_container=True, runtime.append("--log-driver=none") euid, egid = docker_vm_id() - if not onWindows(): # MS Windows does not have getuid() or geteuid() functions + if not onWindows(): + # MS Windows does not have getuid() or geteuid() functions euid, egid = euid or os.geteuid(), egid or os.getgid() - if kwargs.get("no_match_user", None) is False and (euid, egid) != (None, None): + if kwargs.get("no_match_user", None) is False \ + and (euid, egid) != (None, None): runtime.append(u"--user=%d:%d" % (euid, egid)) if rm_container: @@ -447,7 +468,8 @@ def run(self, pull_image=True, rm_container=True, runtime.append(img_id) - self._execute(runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs) + self._execute( + runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs) def _job_popen( diff --git a/cwltool/main.py b/cwltool/main.py index 31a549611..f57cafb6c 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -160,7 +160,9 @@ def arg_parser(): # type: () -> argparse.ArgumentParser parser.add_argument("--js-console", action="store_true", help="Enable javascript console output") parser.add_argument("--user-space-docker-cmd", - help="Specify a user space docker command that will be used to call 'pull' and 'run'") + help="(Linux/OS X only) Specify a user space docker " + "command (like udocker or dx-docker) that will be " + "used to call 'pull' and 'run'") dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS