From 31e6eae8871f0a066bfeb3b5662da99e6f4195ad Mon Sep 17 00:00:00 2001 From: John Fonner Date: Thu, 20 Jul 2017 10:35:42 -0500 Subject: [PATCH 1/2] updated DockerCommandLineJob.run to invoke Singularity instead of Docker. Proof of concept implementation. --- cwltool/job.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cwltool/job.py b/cwltool/job.py index 22b9e1bff..503fd96f1 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -316,10 +316,10 @@ def add_volumes(self, pathmapper, runtime, stage_output): containertgt = vol.target if vol.type in ("File", "Directory"): if not vol.resolved.startswith("_:"): - runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) + runtime.append(u"--bind=%s:%s:ro" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) elif vol.type == "WritableFile": if self.inplace_update: - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) + runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) else: shutil.copy(vol.resolved, vol.target) elif vol.type == "WritableDirectory": @@ -327,14 +327,14 @@ def add_volumes(self, pathmapper, runtime, stage_output): os.makedirs(vol.target, 0o0755) else: if self.inplace_update: - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) + runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt))) else: shutil.copytree(vol.resolved, vol.target) elif vol.type == "CreateFile": createtmp = os.path.join(host_outdir, os.path.basename(vol.target)) with open(createtmp, "wb") as f: f.write(vol.resolved.encode("utf-8")) - runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(createtmp), docker_windows_path_adjust(vol.target))) + runtime.append(u"--bind=%s:%s:ro" % (docker_windows_path_adjust(createtmp), docker_windows_path_adjust(vol.target))) def run(self, pull_image=True, rm_container=True, @@ -370,25 +370,28 @@ def run(self, pull_image=True, rm_container=True, self._setup() - runtime = [u"docker", u"run", u"-i"] + runtime = [u"singularity", u"exec", u"-i"] - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir)) - runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) + runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir)) + runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) self.add_volumes(self.pathmapper, runtime, False) if self.generatemapper: self.add_volumes(self.generatemapper, runtime, True) - runtime.append(u"--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir))) - runtime.append(u"--read-only=true") + runtime.append(u"--pwd=%s" % (docker_windows_path_adjust(self.builder.outdir))) +# runtime.append(u"--read-only=true") # true by default for Singularity images if kwargs.get("custom_net", None) is not None: - runtime.append(u"--net={0}".format(kwargs.get("custom_net"))) - elif kwargs.get("disable_net", None): - runtime.append(u"--net=none") +# runtime.append(u"--net={0}".format(kwargs.get("custom_net"))) + raise UnsupportedRequirement( + "Singularity implementation does not support networking") +# elif kwargs.get("disable_net", None): +# runtime.append(u"--net=none") - if self.stdout: - runtime.append("--log-driver=none") +# No equivalent flag in Singularity +# if self.stdout: +# runtime.append("--log-driver=none") if onWindows(): # windows os dont have getuid or geteuid functions euid = docker_vm_uid() @@ -398,18 +401,19 @@ def run(self, pull_image=True, rm_container=True, if kwargs.get("no_match_user", None) is False and euid is not None: runtime.append(u"--user=%s" % (euid)) - if rm_container: - runtime.append(u"--rm") +# No equivalent +# if rm_container: +# runtime.append(u"--rm") - runtime.append(u"--env=TMPDIR=/tmp") + runtime.prepend(u"SINGULARITYENV_TMPDIR=/tmp") # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") - runtime.append(u"--env=HOME=%s" % self.builder.outdir) + runtime.prepend(u"SINGULARITYENV_HOME=%s" % self.builder.outdir) for t, v in self.environment.items(): - runtime.append(u"--env=%s=%s" % (t, v)) + runtime.prepend(u"SINGULARITYENV_%s=%s" % (t, v)) runtime.append(img_id) From b60fe558ae78e3d0ee823247c224a40cca43487e Mon Sep 17 00:00:00 2001 From: vagrant Date: Fri, 21 Jul 2017 11:47:09 +0000 Subject: [PATCH 2/2] Docker now broken, singu fails with permafail --- cwltool/docker.py | 59 +++++++++++++++++++++++++---------------------- cwltool/job.py | 10 ++++---- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/cwltool/docker.py b/cwltool/docker.py index 9db4dc1e2..9902bb58f 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -18,39 +18,44 @@ def get_image(dockerRequirement, pull_image, dry_run=False): # type: (Dict[Text, Text], bool, bool) -> bool found = False - if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] - for ln in subprocess.check_output( - ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines(): - try: - m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln) - sp = dockerRequirement["dockerImageId"].split(":") - if len(sp) == 1: - sp.append("latest") - elif len(sp) == 2: - # if sp[1] doesn't match valid tag names, it is a part of repository - if not re.match(r'[\w][\w.-]{0,127}', sp[1]): - sp[0] = sp[0] + ":" + sp[1] - sp[1] = "latest" - elif len(sp) == 3: - if re.match(r'[\w][\w.-]{0,127}', sp[2]): - sp[0] = sp[0] + ":" + sp[1] - sp[1] = sp[2] - del sp[2] - - # check for repository:tag match or image id match - if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)): - found = True - break - except ValueError: - pass +#DEBUG: CHECKS IMAGE REGISTRY FOR IDS. SINGULARITY EQ WOULD BE TO CHECK THE FOLDER FOR IMG. + #'node:slim' + imageString = dockerRequirement["dockerPull"].split(":") + imageHit = "{}-{}.img".format(imageString[0], imageString[1]) + if imageHit: + found = True +# for ln in subprocess.check_output( +# ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines(): +# try: +# m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln) +# sp = dockerRequirement["dockerImageId"].split(":") +# if len(sp) == 1: +# sp.append("latest") +# elif len(sp) == 2: +# # if sp[1] doesn't match valid tag names, it is a part of repository +# if not re.match(r'[\w][\w.-]{0,127}', sp[1]): +# sp[0] = sp[0] + ":" + sp[1] +# sp[1] = "latest" +# elif len(sp) == 3: +# if re.match(r'[\w][\w.-]{0,127}', sp[2]): +# sp[0] = sp[0] + ":" + sp[1] +# sp[1] = sp[2] +# del sp[2] +# +# # check for repository:tag match or image id match +# if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)): +# found = True +# break +# except ValueError: +# pass if not found and pull_image: cmd = [] # type: List[Text] if "dockerPull" in dockerRequirement: - cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])] + cmd = ["singularity", "pull", "docker://" + str(dockerRequirement["dockerPull"])] _logger.info(Text(cmd)) if not dry_run: subprocess.check_call(cmd, stdout=sys.stderr) @@ -103,7 +108,7 @@ def get_from_requirements(r, req, pull_image, dry_run=False): if r: errmsg = None try: - subprocess.check_output(["docker", "version"]) + subprocess.check_output(["singularity", "--version"]) except subprocess.CalledProcessError as e: errmsg = "Cannot communicate with docker daemon: " + Text(e) except OSError as e: diff --git a/cwltool/job.py b/cwltool/job.py index 503fd96f1..5fe3ea387 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -348,6 +348,7 @@ def run(self, pull_image=True, rm_container=True, try: env = cast(MutableMapping[Text, Text], os.environ) if docker_req and kwargs.get("use_container") is not False: + print "Req docker" img_id = docker.get_from_requirements(docker_req, True, pull_image) if img_id is None: find_default_container = self.builder.find_default_container @@ -370,7 +371,7 @@ def run(self, pull_image=True, rm_container=True, self._setup() - runtime = [u"singularity", u"exec", u"-i"] + runtime = [u"singularity", u"exec"] runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir)) runtime.append(u"--bind=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp")) @@ -404,16 +405,15 @@ def run(self, pull_image=True, rm_container=True, # No equivalent # if rm_container: # runtime.append(u"--rm") - - runtime.prepend(u"SINGULARITYENV_TMPDIR=/tmp") + runtime.insert(0, u"SINGULARITYENV_TMPDIR=/tmp") # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") - runtime.prepend(u"SINGULARITYENV_HOME=%s" % self.builder.outdir) + runtime.insert(0, u"SINGULARITYENV_HOME=%s" % self.builder.outdir) for t, v in self.environment.items(): - runtime.prepend(u"SINGULARITYENV_%s=%s" % (t, v)) + runtime.insert(0, u"SINGULARITYENV_%s=%s" % (t, v)) runtime.append(img_id)