Handle URLs for command line inputs (#1613)

* Handle URLs for command line inputs

Also handle edge case of an input parameter type is wrapped in
single-item list.

Author: tetron

cwltool/argparser.py

@@ -5,15 +5,18 @@
 from typing import (
     Any,
     AnyStr,
+    Callable,
     Dict,
     List,
     MutableMapping,
     MutableSequence,
     Optional,
     Sequence,
+    Type,
     Union,
     cast,
 )
+import urllib
 
 from schema_salad.ref_resolver import file_uri
 
@@ -716,26 +719,34 @@ class FSAction(argparse.Action):
     objclass = None  # type: str
 
     def __init__(
-        self, option_strings: List[str], dest: str, nargs: Any = None, **kwargs: Any
+        self,
+        option_strings: List[str],
+        dest: str,
+        nargs: Any = None,
+        urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
+        base_uri: str = "",
+        **kwargs: Any,
     ) -> None:
         """Fail if nargs is used."""
         if nargs is not None:
             raise ValueError("nargs not allowed")
+        self.urljoin = urljoin
+        self.base_uri = base_uri
         super().__init__(option_strings, dest, **kwargs)
 
     def __call__(
         self,
         parser: argparse.ArgumentParser,
         namespace: argparse.Namespace,
-        values: Union[AnyStr, Sequence[Any], None],
+        values: Union[str, Sequence[Any], None],
         option_string: Optional[str] = None,
     ) -> None:
         setattr(
             namespace,
             self.dest,
             {
                 "class": self.objclass,
-                "location": file_uri(str(os.path.abspath(cast(AnyStr, values)))),
+                "location": self.urljoin(self.base_uri, cast(str, values)),
             },
         )
 
@@ -744,18 +755,26 @@ class FSAppendAction(argparse.Action):
     objclass = None  # type: str
 
     def __init__(
-        self, option_strings: List[str], dest: str, nargs: Any = None, **kwargs: Any
+        self,
+        option_strings: List[str],
+        dest: str,
+        nargs: Any = None,
+        urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
+        base_uri: str = "",
+        **kwargs: Any,
     ) -> None:
         """Initialize."""
         if nargs is not None:
             raise ValueError("nargs not allowed")
+        self.urljoin = urljoin
+        self.base_uri = base_uri
         super().__init__(option_strings, dest, **kwargs)
 
     def __call__(
         self,
         parser: argparse.ArgumentParser,
         namespace: argparse.Namespace,
-        values: Union[AnyStr, Sequence[Any], None],
+        values: Union[str, Sequence[Any], None],
         option_string: Optional[str] = None,
     ) -> None:
         g = getattr(namespace, self.dest)
@@ -765,7 +784,7 @@ def __call__(
         g.append(
             {
                 "class": self.objclass,
-                "location": file_uri(str(os.path.abspath(cast(AnyStr, values)))),
+                "location": self.urljoin(self.base_uri, cast(str, values)),
             }
         )
 
@@ -794,6 +813,8 @@ def add_argument(
     description: str = "",
     default: Any = None,
     input_required: bool = True,
+    urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
+    base_uri: str = "",
 ) -> None:
     if len(name) == 1:
         flag = "-"
@@ -804,27 +825,32 @@ def add_argument(
     # parameter required.
     required = default is None and input_required
     if isinstance(inptype, MutableSequence):
-        if inptype[0] == "null":
+        if len(inptype) == 1:
+            inptype = inptype[0]
+        elif len(inptype) == 2 and inptype[0] == "null":
+            required = False
+            inptype = inptype[1]
+        elif len(inptype) == 2 and inptype[1] == "null":
             required = False
-            if len(inptype) == 2:
-                inptype = inptype[1]
-            else:
-                _logger.debug("Can't make command line argument from %s", inptype)
-                return None
+            inptype = inptype[0]
+        else:
+            _logger.debug("Can't make command line argument from %s", inptype)
+            return None
 
     ahelp = description.replace("%", "%%")
-    action = None  # type: Optional[Union[argparse.Action, str]]
+    action = None  # type: Optional[Union[Type[argparse.Action], str]]
     atype = None  # type: Any
+    typekw = {}  # type: Dict[str, Any]
 
     if inptype == "File":
-        action = cast(argparse.Action, FileAction)
+        action = FileAction
     elif inptype == "Directory":
-        action = cast(argparse.Action, DirectoryAction)
+        action = DirectoryAction
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "array":
         if inptype["items"] == "File":
-            action = cast(argparse.Action, FileAppendAction)
+            action = FileAppendAction
         elif inptype["items"] == "Directory":
-            action = cast(argparse.Action, DirectoryAppendAction)
+            action = DirectoryAppendAction
         else:
             action = "append"
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
@@ -851,18 +877,20 @@ def add_argument(
         _logger.debug("Can't make command line argument from %s", inptype)
         return None
 
+    if action in (FileAction, DirectoryAction, FileAppendAction, DirectoryAppendAction):
+        typekw["urljoin"] = urljoin
+        typekw["base_uri"] = base_uri
+
     if inptype != "boolean":
-        typekw = {"type": atype}
-    else:
-        typekw = {}
+        typekw["type"] = atype
 
     toolparser.add_argument(
         flag + name,
         required=required,
         help=ahelp,
         action=action,  # type: ignore
         default=default,
-        **typekw
+        **typekw,
     )
 
 
@@ -872,6 +900,8 @@ def generate_parser(
     namemap: Dict[str, str],
     records: List[str],
     input_required: bool = True,
+    urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
+    base_uri: str = "",
 ) -> argparse.ArgumentParser:
     toolparser.description = tool.tool.get("doc", None)
     toolparser.add_argument("job_order", nargs="?", help="Job input json file")
@@ -884,7 +914,15 @@ def generate_parser(
         description = inp.get("doc", "")
         default = inp.get("default", None)
         add_argument(
-            toolparser, name, inptype, records, description, default, input_required
+            toolparser,
+            name,
+            inptype,
+            records,
+            description,
+            default,
+            input_required,
+            urljoin,
+            base_uri,
         )
 
     return toolparser

cwltool/executors.py

@@ -221,6 +221,7 @@ def run_jobs(
                 fsaccess=runtime_context.make_fs_access(""),
             )
             process.parent_wf = process.provenance_object
+
         jobiter = process.job(job_order_object, self.output_callback, runtime_context)
 
         try:

cwltool/main.py

@@ -427,6 +427,8 @@ def init_job_order(
             namemap,
             records,
             input_required,
+            loader.fetcher.urljoin,
+            file_uri(os.getcwd()) + "/",
         )
         if args.tool_help:
             toolparser.print_help(cast(IO[str], stdout))

tests/test_toolargparse.py

@@ -70,6 +70,37 @@
 outputs: []
 """
 
+script_d = """
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: ExpressionTool
+
+inputs:
+  foo:
+    type:
+      - type: enum
+        symbols: [cymbal1, cymbal2]
+
+expression: $(inputs.foo)
+
+outputs: []
+"""
+
+script_e = """
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: ExpressionTool
+
+inputs:
+  foo: File
+
+expression: '{"bar": $(inputs.foo.location)}'
+
+outputs: []
+"""
+
 scripts_argparse_params = [
     ("help", script_a, lambda x: ["--debug", x, "--input", get_data("tests/echo.cwl")]),
     ("boolean", script_b, lambda x: [x, "--help"]),
@@ -79,6 +110,16 @@
         script_c,
         lambda x: [x, "--foo.one", get_data("tests/echo.cwl"), "--foo.two", "test"],
     ),
+    (
+        "foo with d",
+        script_d,
+        lambda x: [x, "--foo", "cymbal2"],
+    ),
+    (
+        "foo with e",
+        script_e,
+        lambda x: [x, "--foo", "http://example.com"],
+    ),
 ]
 
 
@@ -92,7 +133,7 @@ def test_argparse(
         with script_name.open(mode="w") as script:
             script.write(script_contents)
 
-        my_params = ["--outdir", str(tmp_path / "outdir")]
+        my_params = ["--outdir", str(tmp_path / "outdir"), "--debug"]
         my_params.extend(params(script.name))
         assert main(my_params) == 0, name