diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index dd94a13b4..b3240095f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [9, 10, 11, 12, 13, 14] + py-ver-minor: [10, 11, 12, 13, 14] step: [lint, unit, bandit, mypy] env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd9e9105d..bee0e6947 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ Style guide: - PEP-8 (as implemented by the `black` code formatting tool) -- Python 3.8+ compatible code +- Python 3.10+ compatible code - PEP-484 type hints The development is done using `git`, we encourage you to get familiar with it. diff --git a/Makefile b/Makefile index 5c3e31226..28d9b8219 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ shellcheck: FORCE cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py39-plus $^ + pyupgrade --exit-zero-even-if-changed --py310-plus $^ auto-walrus $^ release-test: FORCE diff --git a/README.rst b/README.rst index 7c6ffe22d..9dc97d62a 100644 --- a/README.rst +++ b/README.rst @@ -842,7 +842,7 @@ executor :: executor(tool, job_order_object, runtimeContext, logger) - (Process, Dict[Text, Any], RuntimeContext) -> Tuple[Dict[Text, Any], Text] + (Process, Dict[str, Any], RuntimeContext) -> Tuple[Dict[str, Any], str] An implementation of the top-level workflow execution loop should synchronously run a process object to completion and return the @@ -852,7 +852,7 @@ versionfunc :: () - () -> Text + () -> str Return version string. @@ -879,7 +879,7 @@ resolver :: resolver(document_loader, document) - (Loader, Union[Text, dict[Text, Any]]) -> Text + (Loader, str | dict[str, Any]) -> str Resolve a relative document identifier to an absolute one that can be fetched. @@ -890,7 +890,7 @@ construct_tool_object :: construct_tool_object(toolpath_object, loadingContext) - (MutableMapping[Text, Any], LoadingContext) -> Process + (MutableMapping[str, Any], LoadingContext) -> Process Hook to construct a Process object (eg CommandLineTool) object from a document. @@ -898,7 +898,7 @@ select_resources :: selectResources(request) - (Dict[str, int], RuntimeContext) -> Dict[Text, int] + (Dict[str, int], RuntimeContext) -> Dict[str, int] Take a resource request and turn it into a concrete resource assignment. @@ -906,7 +906,7 @@ make_fs_access :: make_fs_access(basedir) - (Text) -> StdFsAccess + (str) -> StdFsAccess Return a file system access object. @@ -924,6 +924,6 @@ Workflow.make_workflow_step :: make_workflow_step(toolpath_object, pos, loadingContext, parentworkflowProv) - (Dict[Text, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep + (Dict[str, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep Create and return a workflow step object. diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 3f07cea9d..08001c4e7 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,8 +3,8 @@ import argparse import os import urllib -from collections.abc import MutableMapping, MutableSequence, Sequence -from typing import Any, Callable, Optional, Union, cast +from collections.abc import Callable, MutableSequence, Sequence +from typing import Any, cast import rich.markup from rich_argparse import HelpPreviewAction, RichHelpFormatter @@ -732,7 +732,7 @@ def get_default_args() -> dict[str, Any]: class FSAction(argparse.Action): """Base action for our custom actions.""" - objclass: Optional[str] = None + objclass: str | None = None def __init__( self, @@ -754,8 +754,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: setattr( namespace, @@ -770,7 +770,7 @@ def __call__( class FSAppendAction(argparse.Action): """Appending version of the base action for our custom actions.""" - objclass: Optional[str] = None + objclass: str | None = None def __init__( self, @@ -792,8 +792,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: g = getattr(namespace, self.dest) if not g: @@ -808,19 +808,19 @@ def __call__( class FileAction(FSAction): - objclass: Optional[str] = "File" + objclass: str | None = "File" class DirectoryAction(FSAction): - objclass: Optional[str] = "Directory" + objclass: str | None = "Directory" class FileAppendAction(FSAppendAction): - objclass: Optional[str] = "File" + objclass: str | None = "File" class DirectoryAppendAction(FSAppendAction): - objclass: Optional[str] = "Directory" + objclass: str | None = "Directory" class AppendAction(argparse.Action): @@ -844,8 +844,8 @@ def __call__( self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: g = getattr(namespace, self.dest, None) if g is None: @@ -893,59 +893,55 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[type[argparse.Action], str]] = None - atype: Optional[Any] = None + action: type[argparse.Action] | str | None = None + atype: Any | None = None typekw: dict[str, Any] = {} - if inptype == "File": - action = FileAction - elif inptype == "Directory": - action = DirectoryAction - elif isinstance(inptype, MutableMapping) and inptype["type"] == "array": - if inptype["items"] == "File": + match inptype: + case "File": + action = FileAction + case "Directory": + action = DirectoryAction + case {"type": "array", "items": "File"}: action = FileAppendAction - elif inptype["items"] == "Directory": + case {"type": "array", "items": "Directory"}: action = DirectoryAppendAction - else: + case {"type": "array", "items": str(items)}: action = AppendAction - items = inptype["items"] - if items == "int" or items == "long": - atype = int - elif items == "double" or items == "float": - atype = float - elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum": - atype = str - elif isinstance(inptype, MutableMapping) and inptype["type"] == "record": - records.append(name) - for field in inptype["fields"]: - fieldname = name + "." + shortname(field["name"]) - fieldtype = field["type"] - fielddescription = field.get("doc", "") - add_argument( - toolparser, - fieldname, - fieldtype, - records, - fielddescription, - default=default.get(shortname(field["name"]), None) if default else None, - input_required=required, - ) - return - elif inptype == "string": - atype = str - elif inptype == "int": - atype = int - elif inptype == "long": - atype = int - elif inptype == "double": - atype = float - elif inptype == "float": - atype = float - elif inptype == "boolean": - action = "store_true" - else: - _logger.debug("Can't make command line argument from %s", inptype) - return None + match items: + case "int" | "long": + atype = int + case "double" | "float": + atype = float + case {"type": "enum"}: + atype = str + case {"type": "record", "fields": list(fields)}: + records.append(name) + for field in fields: + fieldname = name + "." + shortname(field["name"]) + fieldtype = field["type"] + fielddescription = field.get("doc", "") + add_argument( + toolparser, + fieldname, + fieldtype, + records, + fielddescription, + default=default.get(shortname(field["name"]), None) if default else None, + input_required=required, + ) + return + case "string": + atype = str + case "int" | "long": + atype = int + case "double" | "float": + atype = float + case "boolean": + action = "store_true" + case _: + _logger.debug("Can't make command line argument from %s", inptype) + return None if action in (FileAction, DirectoryAction, FileAppendAction, DirectoryAppendAction): typekw["urljoin"] = urljoin diff --git a/cwltool/builder.py b/cwltool/builder.py index 6b933c03a..cf6d6d5ac 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,9 +3,9 @@ import copy import logging import math -from collections.abc import MutableMapping, MutableSequence +from collections.abc import Callable, MutableMapping, MutableSequence from decimal import Decimal -from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -101,9 +101,9 @@ def __init__( names: Names, requirements: list[CWLObjectType], hints: list[CWLObjectType], - resources: dict[str, Union[int, float]], - mutation_manager: Optional[MutationManager], - formatgraph: Optional[Graph], + resources: dict[str, int | float], + mutation_manager: MutationManager | None, + formatgraph: Graph | None, make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, job_script_provider: Optional["DependenciesConfiguration"], @@ -157,23 +157,27 @@ def __init__( self.pathmapper: Optional["PathMapper"] = None self.prov_obj: Optional["ProvenanceProfile"] = None - self.find_default_container: Optional[Callable[[], str]] = None + self.find_default_container: Callable[[], str] | None = None self.container_engine = container_engine - def build_job_script(self, commands: list[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> str | None: """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None + def _capture_files(self, f: CWLObjectType) -> CWLObjectType: + self.files.append(f) + return f + def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, list[CWLObjectType]], + datum: CWLObjectType | list[CWLObjectType], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, list[int]]] = None, - tail_pos: Optional[Union[str, list[int]]] = None, - ) -> list[MutableMapping[str, Union[str, list[int]]]]: + lead_pos: int | list[int] | None = None, + tail_pos: str | list[int] | None = None, + ) -> list[MutableMapping[str, str | list[int]]]: """ Bind an input object to the command line. @@ -189,8 +193,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: list[MutableMapping[str, Union[str, list[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, str | list[int]]] = [] + binding: MutableMapping[str, str | list[int]] | CommentedMap = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -226,7 +230,7 @@ def bind_input( if isinstance(schema["type"], MutableSequence): bound_input = False for t in schema["type"]: - avsc: Optional[Schema] = None + avsc: Schema | None = None if isinstance(t, str) and self.names.has_name(t, None): avsc = self.names.get_name(t, None) elif ( @@ -360,9 +364,9 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: datum = cast(CWLObjectType, datum) self.files.append(datum) - loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, list[int]]], CWLObjectType - ] = None + loadContents_sourceline: ( + None | MutableMapping[str, str | list[int]] | CWLObjectType + ) = None if binding and binding.get("loadContents"): loadContents_sourceline = binding elif schema.get("loadContents"): @@ -502,7 +506,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, list[str]] = eval_format + evaluated_format: str | list[str] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -555,7 +559,7 @@ def addsf( visit_class( datum.get("secondaryFiles", []), ("File", "Directory"), - _capture_files, + self._capture_files, ) if schema["type"] == "org.w3id.cwl.cwl.Directory": @@ -570,7 +574,7 @@ def addsf( self.files.append(datum) if schema["type"] == "Any": - visit_class(datum, ("File", "Directory"), _capture_files) + visit_class(datum, ("File", "Directory"), self._capture_files) # Position to front of the sort key if binding: @@ -582,30 +586,28 @@ def addsf( return bindings - def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: + def tostr(self, value: MutableMapping[str, str] | Any) -> str: """ Represent an input parameter as a string. :raises WorkflowException: if the item is a File or Directory and the "path" is missing. """ - if isinstance(value, MutableMapping) and value.get("class") in ( - "File", - "Directory", - ): - if "path" not in value: - raise WorkflowException( - '{} object missing "path": {}'.format(value["class"], value) - ) - return value["path"] - elif isinstance(value, ScalarFloat): - rep = RoundTripRepresenter() - dec_value = Decimal(rep.represent_scalar_float(value).value) - if "E" in str(dec_value): - return str(dec_value.quantize(1)) - return str(dec_value) - else: - return str(value) + match value: + case {"class": "File" | "Directory" as class_name, **rest}: + if "path" not in rest: + raise WorkflowException( + '{} object missing "path": {}'.format(class_name, value) + ) + return str(rest["path"]) + case ScalarFloat(): + rep = RoundTripRepresenter() + dec_value = Decimal(rep.represent_scalar_float(value).value) + if "E" in str(dec_value): + return str(dec_value.quantize(1)) + return str(dec_value) + case _: + return str(value) def generate_arg(self, binding: CWLObjectType) -> list[str]: """Convert an input binding to a list of command line arguments.""" @@ -632,30 +634,28 @@ def generate_arg(self, binding: CWLObjectType) -> list[str]: raise WorkflowException("'separate' option can not be specified without prefix") argl: MutableSequence[CWLOutputType] = [] - if isinstance(value, MutableSequence): - if binding.get("itemSeparator") and value: - itemSeparator = cast(str, binding["itemSeparator"]) - argl = [itemSeparator.join([self.tostr(v) for v in value])] - elif binding.get("valueFrom"): - value = [self.tostr(v) for v in value] - return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) - elif prefix and value: + match value: + case MutableSequence(): + if binding.get("itemSeparator") and value: + itemSeparator = cast(str, binding["itemSeparator"]) + argl = [itemSeparator.join([self.tostr(v) for v in value])] + elif binding.get("valueFrom"): + value = [self.tostr(v) for v in value] + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) + elif prefix and value: + return [prefix] + else: + return [] + case {"class": "File" | "Directory"}: + argl = [cast(CWLOutputType, value)] + case MutableMapping(): + return [prefix] if prefix else [] + case True if prefix: return [prefix] - else: + case bool() | None: return [] - elif isinstance(value, MutableMapping) and value.get("class") in ( - "File", - "Directory", - ): - argl = cast(MutableSequence[CWLOutputType], [value]) - elif isinstance(value, MutableMapping): - return [prefix] if prefix else [] - elif value is True and prefix: - return [prefix] - elif value is False or value is None or (value is True and not prefix): - return [] - else: - argl = [value] + case _: + argl = [value] args = [] for j in argl: @@ -668,11 +668,11 @@ def generate_arg(self, binding: CWLObjectType) -> list[str]: def do_eval( self, - ex: Optional[CWLOutputType], - context: Optional[Any] = None, + ex: CWLOutputType | None, + context: Any | None = None, recursive: bool = False, strip_whitespace: bool = True, - ) -> Optional[CWLOutputType]: + ) -> CWLOutputType | None: if recursive: if isinstance(ex, MutableMapping): return {k: self.do_eval(v, context, recursive) for k, v in ex.items()} diff --git a/cwltool/checker.py b/cwltool/checker.py index b3637db20..d81642e1d 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -23,9 +23,9 @@ def _get_type(tp: Any) -> Any: def check_types( srctype: SinkType, sinktype: SinkType, - linkMerge: Optional[str], - valueFrom: Optional[str], -) -> Union[Literal["pass"], Literal["warning"], Literal["exception"]]: + linkMerge: str | None, + valueFrom: str | None, +) -> Literal["pass"] | Literal["warning"] | Literal["exception"]: """ Check if the source and sink types are correct. @@ -33,34 +33,40 @@ def check_types( """ if valueFrom is not None: return "pass" - if linkMerge is None: - if can_assign_src_to_sink(srctype, sinktype, strict=True): - return "pass" - if can_assign_src_to_sink(srctype, sinktype, strict=False): - return "warning" - return "exception" - if linkMerge == "merge_nested": - return check_types( - {"items": _get_type(srctype), "type": "array"}, - _get_type(sinktype), - None, - None, - ) - if linkMerge == "merge_flattened": - return check_types(merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None) - raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") + match linkMerge: + case None: + if can_assign_src_to_sink(srctype, sinktype, strict=True): + return "pass" + if can_assign_src_to_sink(srctype, sinktype, strict=False): + return "warning" + return "exception" + case "merge_nested": + return check_types( + {"items": _get_type(srctype), "type": "array"}, + _get_type(sinktype), + None, + None, + ) + case "merge_flattened": + return check_types( + merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None + ) + case _: + raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") def merge_flatten_type(src: SinkType) -> CWLOutputType: """Return the merge flattened type of the source type.""" - if isinstance(src, MutableSequence): - return [merge_flatten_type(t) for t in src] - if isinstance(src, MutableMapping) and src.get("type") == "array": - return src - return {"items": src, "type": "array"} + match src: + case MutableSequence(): + return [merge_flatten_type(t) for t in src] + case {"type": "array"}: + return src + case _: + return {"items": src, "type": "array"} -def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool = False) -> bool: +def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool: """ Check for identical type specifications, ignoring extra keys like inputBinding. @@ -322,14 +328,14 @@ class _SrcSink(NamedTuple): src: CWLObjectType sink: CWLObjectType - linkMerge: Optional[str] - message: Optional[str] + linkMerge: str | None + message: str | None def _check_all_types( src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], - sourceField: Union[Literal["source"], Literal["outputSource"]], + sourceField: Literal["source"] | Literal["outputSource"], param_to_step: dict[str, CWLObjectType], ) -> dict[str, list[_SrcSink]]: """ @@ -350,7 +356,7 @@ def _check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge: Optional[str] = cast( + linkMerge: str | None = cast( Optional[str], sink.get( "linkMerge", @@ -518,7 +524,7 @@ def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) - def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" - source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) + source_step: MutableMapping[str, Any] | None = param_to_step.get(parm_id) if source_step is not None: if ( source_step.get("loop") is not None diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index a187f0598..54913b47f 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -12,11 +12,17 @@ import threading import urllib import urllib.parse -from collections.abc import Generator, Mapping, MutableMapping, MutableSequence +from collections.abc import ( + Generator, + Iterable, + Mapping, + MutableMapping, + MutableSequence, +) from enum import Enum from functools import cmp_to_key, partial from re import Pattern -from typing import TYPE_CHECKING, Any, Iterable, Optional, TextIO, Union, cast +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -148,11 +154,11 @@ def __init__( self, builder: Builder, script: str, - output_callback: Optional[OutputCallbackType], + output_callback: OutputCallbackType | None, requirements: list[CWLObjectType], hints: list[CWLObjectType], - outdir: Optional[str] = None, - tmpdir: Optional[str] = None, + outdir: str | None = None, + tmpdir: str | None = None, ) -> None: """Initialize this ExpressionJob.""" self.builder = builder @@ -167,7 +173,7 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: try: normalizeFilesDirs(self.builder.job) @@ -201,7 +207,7 @@ class ExpressionTool(Process): def job( self, job_order: CWLObjectType, - output_callbacks: Optional[OutputCallbackType], + output_callbacks: OutputCallbackType | None, runtimeContext: RuntimeContext, ) -> Generator[ExpressionJob, None, None]: builder = self._init_job(job_order, runtimeContext) @@ -221,7 +227,7 @@ class AbstractOperation(Process): def job( self, job_order: CWLObjectType, - output_callbacks: Optional[OutputCallbackType], + output_callbacks: OutputCallbackType | None, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: raise WorkflowException("Abstract operation cannot be executed.") @@ -233,7 +239,7 @@ def remove_path(f: CWLObjectType) -> None: del f["path"] -def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWLObjectType]: +def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectType | None: """ Remap a file from internal path to external path. @@ -312,7 +318,7 @@ class CallbackJob: def __init__( self, job: "CommandLineTool", - output_callback: Optional[OutputCallbackType], + output_callback: OutputCallbackType | None, cachebuilder: Builder, jobcache: str, ) -> None: @@ -321,12 +327,12 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if self.output_callback: self.output_callback( @@ -470,7 +476,7 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) - def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None: + def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: """ Test and initialize the working directory. @@ -797,7 +803,7 @@ def job( job_order: CWLObjectType, output_callbacks: OutputCallbackType, runtimeContext: RuntimeContext, - ) -> Generator[Union[JobBase, CallbackJob], None, None]: + ) -> Generator[JobBase | CallbackJob, None, None]: workReuse, _ = self.get_requirement("WorkReuse") enableReuse = workReuse.get("enableReuse", True) if workReuse else True @@ -840,15 +846,13 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict: dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] = { - "cmdline": cmdline - } + keydict: dict[str, MutableSequence[str | int] | CWLObjectType] = {"cmdline": cmdline} for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: keydict[shortcut] = self.tool[shortcut] - def calc_checksum(location: str) -> Optional[str]: + def calc_checksum(location: str) -> str | None: for e in cachebuilder.files: if ( "location" in e @@ -943,7 +947,7 @@ def remove_prefix(s: str, prefix: str) -> str: def update_status_output_callback( output_callbacks: OutputCallbackType, jobcachelock: TextIO, - outputs: Optional[CWLObjectType], + outputs: CWLObjectType | None, processStatus: str, ) -> None: # save status to the lockfile then release the lock @@ -1198,13 +1202,13 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, set[CWLObjectType]], + ports: CommentedSeq | set[CWLObjectType], builder: Builder, outdir: str, rcode: int, compute_checksum: bool = True, jobname: str = "", - readers: Optional[MutableMapping[str, CWLObjectType]] = None, + readers: MutableMapping[str, CWLObjectType] | None = None, ) -> OutputPortsType: ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) @@ -1280,11 +1284,11 @@ def collect_output( outdir: str, fs_access: StdFsAccess, compute_checksum: bool = True, - ) -> Optional[CWLOutputType]: + ) -> CWLOutputType | None: r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) - result: Optional[CWLOutputType] = None + result: CWLOutputType | None = None if "outputBinding" in schema: binding = cast( MutableMapping[str, Union[bool, str, list[str]]], @@ -1384,13 +1388,14 @@ def collect_output( optional = False single = False - if isinstance(schema["type"], MutableSequence): - if "null" in schema["type"]: - optional = True - if "File" in schema["type"] or "Directory" in schema["type"]: + match schema["type"]: + case MutableSequence(): + if "null" in schema["type"]: + optional = True + if "File" in schema["type"] or "Directory" in schema["type"]: + single = True + case "File" | "Directory": single = True - elif schema["type"] == "File" or schema["type"] == "Directory": - single = True if "outputEval" in binding: with SourceLine(binding, "outputEval", WorkflowException, debug): diff --git a/cwltool/context.py b/cwltool/context.py index 83dd5a190..4e106ac48 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,8 +5,8 @@ import shutil import tempfile import threading -from collections.abc import Iterable -from typing import IO, TYPE_CHECKING, Any, Callable, Literal, Optional, TextIO, Union +from collections.abc import Callable, Iterable +from typing import IO, TYPE_CHECKING, Any, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -35,7 +35,7 @@ class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -54,8 +54,8 @@ def make_tool_notimpl(toolpath_object: CommentedMap, loadingContext: "LoadingCon def log_handler( outdir: str, base_path_logs: str, - stdout_path: Optional[str], - stderr_path: Optional[str], + stdout_path: str | None, + stderr_path: str | None, ) -> None: """Move logs from log location to final output.""" if outdir != base_path_logs: @@ -76,31 +76,31 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[list[CWLObjectType]] = None - self.hints: Optional[list[CWLObjectType]] = None + self.requirements: list[CWLObjectType] | None = None + self.hints: list[CWLObjectType] | None = None self.overrides_list: list[CWLObjectType] = [] - self.loader: Optional[Loader] = None - self.avsc_names: Optional[Names] = None + self.loader: Loader | None = None + self.avsc_names: Names | None = None self.disable_js_validation: bool = False - self.js_hint_options_file: Optional[str] = None + self.js_hint_options_file: str | None = None self.do_validate: bool = True self.enable_dev: bool = False self.strict: bool = True - self.resolver: Optional[ResolverType] = None - self.fetcher_constructor: Optional[FetcherCallableType] = None + self.resolver: ResolverType | None = None + self.fetcher_constructor: FetcherCallableType | None = None self.construct_tool_object = default_make_tool - self.research_obj: Optional[ResearchObject] = None + self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" self.host_provenance: bool = False self.user_provenance: bool = False self.prov_obj: Optional["ProvenanceProfile"] = None - self.do_update: Optional[bool] = None - self.jobdefaults: Optional[CommentedMap] = None + self.do_update: bool | None = None + self.jobdefaults: CommentedMap | None = None self.doc_cache: bool = True self.relax_path_checks: bool = False self.singularity: bool = False @@ -119,24 +119,24 @@ def copy(self) -> "LoadingContext": class RuntimeContext(ContextBase): - outdir: Optional[str] = None + outdir: str | None = None tmpdir: str = "" tmpdir_prefix: str = DEFAULT_TMP_PREFIX tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: + def __init__(self, kwargs: dict[str, Any] | None = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ [dict[str, Union[int, float]], RuntimeContext], dict[str, Union[int, float]], ] - self.user_space_docker_cmd: Optional[str] = None + self.user_space_docker_cmd: str | None = None self.secret_store: Optional["SecretStore"] = None self.no_read_only: bool = False - self.custom_net: Optional[str] = None + self.custom_net: str | None = None self.no_match_user: bool = False - self.preserve_environment: Optional[Iterable[str]] = None + self.preserve_environment: Iterable[str] | None = None self.preserve_entire_environment: bool = False self.use_container: bool = True self.force_docker_pull: bool = False @@ -144,7 +144,7 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.rm_tmpdir: bool = True self.pull_image: bool = True self.rm_container: bool = True - self.move_outputs: Union[Literal["move"], Literal["leave"], Literal["copy"]] = "move" + self.move_outputs: Literal["move"] | Literal["leave"] | Literal["copy"] = "move" self.log_dir: str = "" self.set_log_dir = set_log_dir self.log_dir_handler = log_handler @@ -155,9 +155,9 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.debug: bool = False self.compute_checksum: bool = True self.name: str = "" - self.default_container: Optional[str] = "" - self.find_default_container: Optional[Callable[[HasReqsHints], Optional[str]]] = None - self.cachedir: Optional[str] = None + self.default_container: str | None = "" + self.find_default_container: Callable[[HasReqsHints], str | None] | None = None + self.cachedir: str | None = None self.part_of: str = "" self.basedir: str = "" self.toplevel: bool = False @@ -169,32 +169,32 @@ def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: self.docker_tmpdir: str = "" self.docker_stagedir: str = "" self.js_console: bool = False - self.job_script_provider: Optional[DependenciesConfiguration] = None - self.select_resources: Optional[select_resources_callable] = None + self.job_script_provider: DependenciesConfiguration | None = None + self.select_resources: select_resources_callable | None = None self.eval_timeout: float = 60 - self.postScatterEval: Optional[Callable[[CWLObjectType], Optional[CWLObjectType]]] = None - self.on_error: Union[Literal["stop"], Literal["continue"]] = "stop" + self.postScatterEval: Callable[[CWLObjectType], CWLObjectType | None] | None = None + self.on_error: Literal["stop"] | Literal["continue"] = "stop" self.strict_memory_limit: bool = False self.strict_cpu_limit: bool = False - self.cidfile_dir: Optional[str] = None - self.cidfile_prefix: Optional[str] = None + self.cidfile_dir: str | None = None + self.cidfile_prefix: str | None = None - self.workflow_eval_lock: Optional[threading.Condition] = None - self.research_obj: Optional[ResearchObject] = None + self.workflow_eval_lock: Union[threading.Condition, None] = None + self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" - self.process_run_id: Optional[str] = None + self.process_run_id: str | None = None self.prov_host: bool = False self.prov_user: bool = False - self.prov_obj: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None self.mpi_config: MpiConfig = MpiConfig() - self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None - self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None + self.default_stdout: IO[bytes] | TextIO | None = None + self.default_stderr: IO[bytes] | TextIO | None = None self.validate_only: bool = False self.validate_stdout: Optional["SupportsWrite[str]"] = None - self.workflow_job_step_name_callback: Optional[ + self.workflow_job_step_name_callback: None | ( Callable[[WorkflowJobStep, CWLObjectType], str] - ] = None + ) = None super().__init__(kwargs) if self.tmp_outdir_prefix == "": diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 5135eaa46..86dcb3cdd 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,7 +2,6 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Union from .loghandler import _logger from .utils import CWLObjectType @@ -11,7 +10,7 @@ def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: - out: Union[str, bytes] = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec + out: str | bytes = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec except Exception as e: _logger.debug("Error checking CUDA version with nvidia-smi: %s", e, exc_info=e) return ("", 0) diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index a09a57c34..a9ca171e1 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -5,8 +5,9 @@ import pwd import re import uuid +from collections.abc import Callable from getpass import getuser -from typing import IO, Any, Callable, Optional, TypedDict, Union +from typing import IO, Any, Optional, TypedDict, Union def _whoami() -> tuple[str, str]: @@ -48,7 +49,7 @@ def _check_mod_11_2(numeric_string: str) -> bool: return nums[-1].upper() == checkdigit -def _valid_orcid(orcid: Optional[str]) -> str: +def _valid_orcid(orcid: str | None) -> str: """ Ensure orcid is a valid ORCID identifier. @@ -115,27 +116,27 @@ def _valid_orcid(orcid: Optional[str]) -> str: class Aggregate(TypedDict, total=False): """RO Aggregate class.""" - uri: Optional[str] - bundledAs: Optional[dict[str, Any]] - mediatype: Optional[str] - conformsTo: Optional[Union[str, list[str]]] - createdOn: Optional[str] - createdBy: Optional[dict[str, str]] + uri: str | None + bundledAs: dict[str, Any] | None + mediatype: str | None + conformsTo: str | list[str] | None + createdOn: str | None + createdBy: dict[str, str] | None # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported class AuthoredBy(TypedDict, total=False): """RO AuthoredBy class.""" - orcid: Optional[str] - name: Optional[str] - uri: Optional[str] + orcid: str | None + name: str | None + uri: str | None def checksum_copy( src_file: IO[Any], - dst_file: Optional[IO[Any]] = None, - hasher: Optional[Callable[[], "hashlib._Hash"]] = None, + dst_file: IO[Any] | None = None, + hasher: Callable[[], "hashlib._Hash"] | None = None, buffersize: int = 1024 * 1024, ) -> str: """Compute checksums while copying a file.""" diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index e2208378f..c0aa5ec5b 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,7 +6,7 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -41,7 +41,7 @@ from .ro import ResearchObject -def copy_job_order(job: Union[Process, JobsType], job_order_object: CWLObjectType) -> CWLObjectType: +def copy_job_order(job: Process | JobsType, job_order_object: CWLObjectType) -> CWLObjectType: """Create copy of job object for provenance.""" if not isinstance(job, WorkflowJob): # direct command line tool execution @@ -78,7 +78,7 @@ def __init__( user_provenance: bool, orcid: str, fsaccess: StdFsAccess, - run_uuid: Optional[uuid.UUID] = None, + run_uuid: uuid.UUID | None = None, ) -> None: """Initialize the provenance profile.""" self.fsaccess = fsaccess @@ -198,8 +198,8 @@ def evaluate( self.used_artefacts(customised_job, self.workflow_run_uri) def record_process_start( - self, process: Process, job: JobsType, process_run_id: Optional[str] = None - ) -> Optional[str]: + self, process: Process, job: JobsType, process_run_id: str | None = None + ) -> str | None: if not hasattr(process, "steps"): process_run_id = self.workflow_run_uri elif not hasattr(job, "workflow"): @@ -215,7 +215,7 @@ def start_process( self, process_name: str, when: datetime.datetime, - process_run_id: Optional[str] = None, + process_run_id: str | None = None, ) -> str: """Record the start of each Process.""" if process_run_id is None: @@ -237,7 +237,7 @@ def record_process_end( self, process_name: str, process_run_id: str, - outputs: Union[CWLObjectType, MutableSequence[CWLObjectType], None], + outputs: CWLObjectType | MutableSequence[CWLObjectType] | None, when: datetime.datetime, ) -> None: self.generate_output_prov(outputs, process_run_id, process_name) @@ -248,7 +248,7 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename - entity: Optional[ProvEntity] = None + entity: ProvEntity | None = None checksum = None if "checksum" in value: csum = cast(str, value["checksum"]) @@ -352,10 +352,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: list[tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[str | Identifier, Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: list[tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[str | Identifier, Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -436,127 +436,126 @@ def declare_string(self, value: str) -> tuple[ProvEntity, str]: def declare_artefact(self, value: Any) -> ProvEntity: """Create data artefact entities for all file objects.""" - if value is None: - # FIXME: If this can happen in CWL, we'll - # need a better way to represent this in PROV - return self.document.entity(CWLPROV["None"], {PROV_LABEL: "None"}) - - if isinstance(value, (bool, int, float)): - # Typically used in job documents for flags - - # FIXME: Make consistent hash URIs for these - # that somehow include the type - # (so "1" != 1 != "1.0" != true) - entity = self.document.entity(uuid.uuid4().urn, {PROV_VALUE: value}) - self.research_object.add_uri(entity.identifier.uri) - return entity - - if isinstance(value, str): - (entity, _) = self.declare_string(value) - return entity - - if isinstance(value, bytes): - # If we got here then we must be in Python 3 - byte_s = BytesIO(value) - data_file = self.research_object.add_data_file(byte_s) - # FIXME: Don't naively assume add_data_file uses hash in filename! - data_id = f"data:{PurePosixPath(data_file).stem}" - return self.document.entity( - data_id, - {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(value)}, - ) + if isinstance(value, MutableMapping) and "@id" in value: + # Already processed this value, but it might not be in this PROV + entities = self.document.get_record(value["@id"]) + if entities: + return cast(list[ProvEntity], entities)[0] + # else, unknown in PROV, re-add below as if it's fresh + + match value: + case None: + # FIXME: If this can happen in CWL, we'll + # need a better way to represent this in PROV + return self.document.entity(CWLPROV["None"], {PROV_LABEL: "None"}) + + case bool() | int() | float(): + # Typically used in job documents for flags + + # FIXME: Make consistent hash URIs for these + # that somehow include the type + # (so "1" != 1 != "1.0" != true) + entity = self.document.entity(uuid.uuid4().urn, {PROV_VALUE: value}) + self.research_object.add_uri(entity.identifier.uri) + return entity - if isinstance(value, MutableMapping): - if "@id" in value: - # Already processed this value, but it might not be in this PROV - entities = self.document.get_record(value["@id"]) - if entities: - return cast(list[ProvEntity], entities)[0] - # else, unknown in PROV, re-add below as if it's fresh + case str(val): + return self.declare_string(val)[0] + + case bytes(val): + # If we got here then we must be in Python 3 + byte_s = BytesIO(val) + data_file = self.research_object.add_data_file(byte_s) + # FIXME: Don't naively assume add_data_file uses hash in filename! + data_id = f"data:{PurePosixPath(data_file).stem}" + return self.document.entity( + data_id, + {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(val)}, + ) # Base case - we found a File we need to update - if value.get("class") == "File": - (entity, _, _) = self.declare_file(value) + case {"class": "File"}: + entity = self.declare_file(value)[0] value["@id"] = entity.identifier.uri return entity - - if value.get("class") == "Directory": + case {"class": "Directory"}: entity = self.declare_directory(value) value["@id"] = entity.identifier.uri return entity - coll_id = value.setdefault("@id", uuid.uuid4().urn) - # some other kind of dictionary? - # TODO: also Save as JSON - coll = self.document.entity( - coll_id, - [ - (PROV_TYPE, WFPROV["Artifact"]), - (PROV_TYPE, PROV["Collection"]), - (PROV_TYPE, PROV["Dictionary"]), - ], - ) + case {**rest}: + coll_id = value.setdefault("@id", uuid.uuid4().urn) + # some other kind of dictionary? + # TODO: also Save as JSON + coll = self.document.entity( + coll_id, + [ + (PROV_TYPE, WFPROV["Artifact"]), + (PROV_TYPE, PROV["Collection"]), + (PROV_TYPE, PROV["Dictionary"]), + ], + ) - if value.get("class"): - _logger.warning("Unknown data class %s.", value["class"]) - # FIXME: The class might be "http://example.com/somethingelse" - coll.add_asserted_type(CWLPROV[value["class"]]) - - # Let's iterate and recurse - coll_attribs: list[tuple[Union[str, Identifier], Any]] = [] - for key, val in value.items(): - v_ent = self.declare_artefact(val) - self.document.membership(coll, v_ent) - m_entity = self.document.entity(uuid.uuid4().urn) - # Note: only support PROV-O style dictionary - # https://www.w3.org/TR/prov-dictionary/#dictionary-ontological-definition - # as prov.py do not easily allow PROV-N extensions - m_entity.add_asserted_type(PROV["KeyEntityPair"]) - m_entity.add_attributes({PROV["pairKey"]: str(key), PROV["pairEntity"]: v_ent}) - coll_attribs.append((PROV["hadDictionaryMember"], m_entity)) - coll.add_attributes(coll_attribs) - self.research_object.add_uri(coll.identifier.uri) - return coll - - # some other kind of Collection? - # TODO: also save as JSON - try: - members = [] - for each_input_obj in iter(value): - # Recurse and register any nested objects - e = self.declare_artefact(each_input_obj) - members.append(e) - - # If we reached this, then we were allowed to iterate - coll = self.document.entity( - uuid.uuid4().urn, - [ - (PROV_TYPE, WFPROV["Artifact"]), - (PROV_TYPE, PROV["Collection"]), - ], - ) - if not members: - coll.add_asserted_type(PROV["EmptyCollection"]) - else: - for member in members: - # FIXME: This won't preserve order, for that - # we would need to use PROV.Dictionary - # with numeric keys - self.document.membership(coll, member) - self.research_object.add_uri(coll.identifier.uri) - # FIXME: list value does not support adding "@id" - return coll - except TypeError: - _logger.warning("Unrecognized type %s of %r", type(value), value, exc_info=True) - # Let's just fall back to Python repr() - entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) - self.research_object.add_uri(entity.identifier.uri) - return entity + if rest.get("class"): + _logger.warning("Unknown data class %s.", rest["class"]) + # FIXME: The class might be "http://example.com/somethingelse" + coll.add_asserted_type(CWLPROV[str(rest["class"])]) + + # Let's iterate and recurse + coll_attribs: list[tuple[str | Identifier, Any]] = [] + for key, kval in rest.items(): + v_ent = self.declare_artefact(kval) + self.document.membership(coll, v_ent) + m_entity = self.document.entity(uuid.uuid4().urn) + # Note: only support PROV-O style dictionary + # https://www.w3.org/TR/prov-dictionary/#dictionary-ontological-definition + # as prov.py do not easily allow PROV-N extensions + m_entity.add_asserted_type(PROV["KeyEntityPair"]) + m_entity.add_attributes({PROV["pairKey"]: str(key), PROV["pairEntity"]: v_ent}) + coll_attribs.append((PROV["hadDictionaryMember"], m_entity)) + coll.add_attributes(coll_attribs) + self.research_object.add_uri(coll.identifier.uri) + return coll + + case _: # some other kind of Collection? + # TODO: also save as JSON + try: + members = [] + for each_input_obj in iter(value): + # Recurse and register any nested objects + e = self.declare_artefact(each_input_obj) + members.append(e) + + # If we reached this, then we were allowed to iterate + coll = self.document.entity( + uuid.uuid4().urn, + [ + (PROV_TYPE, WFPROV["Artifact"]), + (PROV_TYPE, PROV["Collection"]), + ], + ) + if not members: + coll.add_asserted_type(PROV["EmptyCollection"]) + else: + for member in members: + # FIXME: This won't preserve order, for that + # we would need to use PROV.Dictionary + # with numeric keys + self.document.membership(coll, member) + self.research_object.add_uri(coll.identifier.uri) + # FIXME: list value does not support adding "@id" + return coll + except TypeError: + _logger.warning("Unrecognized type %s of %r", type(value), value, exc_info=True) + # Let's just fall back to Python repr() + entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) + self.research_object.add_uri(entity.identifier.uri) + return entity def used_artefacts( self, - job_order: Union[CWLObjectType, list[CWLObjectType]], + job_order: CWLObjectType | list[CWLObjectType], process_run_id: str, - name: Optional[str] = None, + name: str | None = None, ) -> None: """Add used() for each data artefact.""" if isinstance(job_order, list): @@ -583,9 +582,9 @@ def used_artefacts( def generate_output_prov( self, - final_output: Union[CWLObjectType, MutableSequence[CWLObjectType], None], - process_run_id: Optional[str], - name: Optional[str], + final_output: CWLObjectType | MutableSequence[CWLObjectType] | None, + process_run_id: str | None, + name: str | None, ) -> None: """Call wasGeneratedBy() for each output,copy the files into the RO.""" if isinstance(final_output, MutableSequence): @@ -657,7 +656,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: list[tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[str | Identifier, Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -666,7 +665,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> list[QualifiedName]: + def finalize_prov_profile(self, name: str | None) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 5bc846dac..28b7c86df 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -10,7 +10,7 @@ from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath from socket import getfqdn -from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, cast import prov.model as provM from prov.model import ProvDocument @@ -95,7 +95,7 @@ def initialize_provenance( user_provenance: bool, orcid: str, fsaccess: StdFsAccess, - run_uuid: Optional[uuid.UUID] = None, + run_uuid: uuid.UUID | None = None, ) -> "ProvenanceProfile": """ Provide a provenance profile initialization hook function. @@ -233,7 +233,7 @@ def host_provenance(self, document: ProvDocument) -> None: }, ) - def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) -> None: + def add_tagfile(self, path: str, timestamp: datetime.datetime | None = None) -> None: """Add tag files to our research object.""" self.self_check() checksums = {} @@ -273,9 +273,9 @@ def _ro_aggregates(self) -> list[Aggregate]: def guess_mediatype( rel_path: str, - ) -> tuple[Optional[str], Optional[Union[str, list[str]]]]: + ) -> tuple[str | None, str | list[str] | None]: """Return the mediatypes.""" - media_types: dict[Union[str, None], str] = { + media_types: dict[str | None, str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -289,7 +289,7 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: dict[Union[str, None], str] = { + conforms_to: dict[str | None, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } @@ -304,13 +304,13 @@ def guess_mediatype( "json": "http://www.w3.org/Submission/2013/SUBM-prov-json-20130424/", } - extension: Optional[str] = rel_path.rsplit(".", 1)[-1].lower() + extension: str | None = rel_path.rsplit(".", 1)[-1].lower() if extension == rel_path: # No ".", no extension extension = None - mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, list[str]]] = conforms_to.get(extension, None) + mediatype: str | None = media_types.get(extension, None) + conformsTo: str | list[str] | None = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -401,7 +401,7 @@ def guess_mediatype( aggregates.extend(self._external_aggregates) return aggregates - def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Aggregate: + def add_uri(self, uri: str, timestamp: datetime.datetime | None = None) -> Aggregate: """Add the given URI to this RO.""" self.self_check() aggr: Aggregate = {"uri": uri} @@ -481,7 +481,7 @@ def _ro_annotations(self) -> list[Annotation]: annotations.extend(self.annotations) return annotations - def _authored_by(self) -> Optional[AuthoredBy]: + def _authored_by(self) -> AuthoredBy | None: authored_by: AuthoredBy = {} """Returns the authoredBy metadata if it was supplied on CLI""" if self.orcid: @@ -536,8 +536,8 @@ def has_data_file(self, sha1hash: str) -> bool: def add_data_file( self, from_fp: IO[Any], - timestamp: Optional[datetime.datetime] = None, - content_type: Optional[str] = None, + timestamp: datetime.datetime | None = None, + content_type: str | None = None, ) -> str: """Copy inputs to data/ folder.""" self.self_check() @@ -577,7 +577,7 @@ def add_data_file( return rel_path def _self_made( - self, timestamp: Optional[datetime.datetime] = None + self, timestamp: datetime.datetime | None = None ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() @@ -635,7 +635,7 @@ def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: def _relativise_files( self, - structure: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType]], + structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], ) -> None: """Save any file objects into the RO and update the local paths.""" # Base case - we found a File we need to update @@ -643,7 +643,7 @@ def _relativise_files( if isinstance(structure, MutableMapping): if structure.get("class") == "File": - relative_path: Optional[Union[str, PurePosixPath]] = None + relative_path: str | PurePosixPath | None = None if "checksum" in structure: raw_checksum = cast(str, structure["checksum"]) alg, checksum = raw_checksum.split("$") diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index 06d7d0bf7..ecd6463d6 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -12,7 +12,7 @@ from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Optional, Union, cast +from typing import Any, BinaryIO, cast from schema_salad.utils import json_dumps @@ -98,7 +98,7 @@ def readable(self) -> bool: """Return False, reading is not supported.""" return False - def truncate(self, size: Optional[int] = None) -> int: + def truncate(self, size: int | None = None) -> int: """Resize the stream, only if we haven't started writing.""" # FIXME: This breaks contract IOBase, # as it means we would have to recalculate the hash @@ -108,8 +108,8 @@ def truncate(self, size: Optional[int] = None) -> int: def write_bag_file( - research_object: "ResearchObject", path: str, encoding: Optional[str] = ENCODING -) -> Union[TextIOWrapper, WritableBagFile]: + research_object: "ResearchObject", path: str, encoding: str | None = ENCODING +) -> TextIOWrapper | WritableBagFile: """Write the bag file into our research object.""" research_object.self_check() # For some reason below throws BlockingIOError @@ -123,7 +123,7 @@ def write_bag_file( def open_log_file_for_activity( research_object: "ResearchObject", uuid_uri: str -) -> Union[TextIOWrapper, WritableBagFile]: +) -> TextIOWrapper | WritableBagFile: """Begin the per-activity log.""" research_object.self_check() # Ensure valid UUID for safe filenames @@ -196,7 +196,7 @@ def _finalize(research_object: "ResearchObject") -> None: (Path(research_object.folder) / "manifest-sha1.txt").touch() -def close_ro(research_object: "ResearchObject", save_to: Optional[str] = None) -> None: +def close_ro(research_object: "ResearchObject", save_to: str | None = None) -> None: """Close the Research Object, optionally saving to specified folder. Closing will remove any temporary files used by this research object. diff --git a/cwltool/docker.py b/cwltool/docker.py index b03ae635c..69fecc830 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,9 +9,9 @@ import subprocess # nosec import sys import threading -from collections.abc import MutableMapping +from collections.abc import Callable, MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Optional, cast +from typing import Optional, cast import requests @@ -26,7 +26,7 @@ _IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[list[str]] = None +__docker_machine_mounts: list[str] | None = None __docker_machine_mounts_lock = threading.Lock() @@ -54,7 +54,7 @@ def _get_docker_machine_mounts() -> list[str]: return __docker_machine_mounts -def _check_docker_machine_path(path: Optional[str]) -> None: +def _check_docker_machine_path(path: str | None) -> None: if path is None: return mounts = _get_docker_machine_mounts() @@ -201,7 +201,7 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") @@ -234,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -245,7 +245,7 @@ def add_writable_file_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable file mapping to the runtime option list.""" @@ -269,7 +269,7 @@ def add_writable_directory_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable directory mapping to the runtime option list.""" @@ -307,7 +307,7 @@ def _required_env(self) -> dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> tuple[list[str], Optional[str]]: + ) -> tuple[list[str], str | None]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -374,7 +374,7 @@ def create_runtime( if self.builder.resources.get("cudaDeviceCount"): runtime.append("--gpus=" + str(self.builder.resources["cudaDeviceCount"])) - cidfile_path: Optional[str] = None + cidfile_path: str | None = None # add parameters to docker to write a container ID file if runtimeContext.user_space_docker_cmd is None: if runtimeContext.cidfile_dir: diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index 90484b686..2f93934cc 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,9 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import Optional -def docker_vm_id() -> tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[int | None, int | None]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +20,7 @@ def docker_vm_id() -> tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: list[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> str | None: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -39,7 +38,7 @@ def check_output_and_strip(cmd: list[str]) -> Optional[str]: return None -def docker_machine_name() -> Optional[str]: +def docker_machine_name() -> str | None: """ Get the machine name of the active docker-machine machine. @@ -80,7 +79,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: list[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> int | None: """ Run the provided command and returns the integer value of the result. @@ -97,7 +96,7 @@ def cmd_output_to_int(cmd: list[str]) -> Optional[int]: return None -def boot2docker_id() -> tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[int | None, int | None]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +107,7 @@ def boot2docker_id() -> tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[int | None, int | None]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/executors.py b/cwltool/executors.py index 9d0559726..090e4a676 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -9,7 +9,7 @@ from abc import ABCMeta, abstractmethod from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import Optional, Union, cast +from typing import Optional, cast import psutil from mypy_extensions import mypyc_attr @@ -39,7 +39,7 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" - self.final_output: MutableSequence[Optional[CWLObjectType]] = [] + self.final_output: MutableSequence[CWLObjectType | None] = [] self.final_status: list[str] = [] self.output_dirs: set[str] = set() @@ -49,10 +49,10 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> tuple[Optional[CWLObjectType], str]: + ) -> tuple[CWLObjectType | None, str]: return self.execute(process, job_order_object, runtime_context, logger) - def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: + def output_callback(self, out: CWLObjectType | None, process_status: str) -> None: """Collect the final status and outputs.""" self.final_status.append(process_status) self.final_output.append(out) @@ -73,7 +73,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[CWLObjectType | None, str]: """Execute the process.""" self.final_output = [] @@ -102,7 +102,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[list[CWLObjectType]] = None + job_reqs: list[CWLObjectType] | None = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -164,7 +164,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: and isinstance(process, (JobBase, Process, WorkflowJobStep, WorkflowJob)) and process.parent_wf ): - process_run_id: Optional[str] = None + process_run_id: str | None = None name = "primary" process.parent_wf.generate_output_prov(self.final_output[0], process_run_id, name) process.parent_wf.document.wasEndedBy( @@ -189,7 +189,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - process_run_id: Optional[str] = None + process_run_id: str | None = None # define provenance profile for single commandline tool if not isinstance(process, Workflow) and runtime_context.research_obj is not None: @@ -281,10 +281,10 @@ def __init__(self) -> None: self.allocated_cuda: int = 0 def select_resources( - self, request: dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, int | float], runtime_context: RuntimeContext + ) -> dict[str, int | float]: # pylint: disable=unused-argument """NaĂ¯ve check for available cpu cores and memory.""" - result: dict[str, Union[int, float]] = {} + result: dict[str, int | float] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -309,7 +309,7 @@ def select_resources( def _runner( self, - job: Union[JobBase, WorkflowJob, CallbackJob, ExpressionJob], + job: JobBase | WorkflowJob | CallbackJob | ExpressionJob, runtime_context: RuntimeContext, TMPDIR_LOCK: threading.Lock, ) -> None: @@ -346,7 +346,7 @@ def _runner( def run_job( self, - job: Optional[JobsType], + job: JobsType | None, runtime_context: RuntimeContext, ) -> None: """Execute a single Job in a separate thread.""" @@ -485,6 +485,6 @@ def execute( process: Process, job_order_object: CWLObjectType, runtime_context: RuntimeContext, - logger: Optional[logging.Logger] = None, - ) -> tuple[Optional[CWLObjectType], str]: + logger: logging.Logger | None = None, + ) -> tuple[CWLObjectType | None, str]: return {}, "success" diff --git a/cwltool/factory.py b/cwltool/factory.py index 08b57c00c..fc00eb061 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Optional, Union +from typing import Any from . import load_tool from .context import LoadingContext, RuntimeContext @@ -10,7 +10,7 @@ class WorkflowStatus(Exception): - def __init__(self, out: Optional[CWLObjectType], status: str) -> None: + def __init__(self, out: CWLObjectType | None, status: str) -> None: """Signaling exception for the status of a Workflow.""" super().__init__("Completed %s" % status) self.out = out @@ -25,8 +25,7 @@ def __init__(self, t: Process, factory: "Factory") -> None: self.t = t self.factory = factory - def __call__(self, **kwargs): - # type: (**Any) -> Union[str, Optional[CWLObjectType]] + def __call__(self, **kwargs: Any) -> str | CWLObjectType | None: """Invoke the tool.""" runtime_context = self.factory.runtime_context.copy() runtime_context.basedir = os.getcwd() @@ -45,9 +44,9 @@ class Factory: def __init__( self, - executor: Optional[JobExecutor] = None, - loading_context: Optional[LoadingContext] = None, - runtime_context: Optional[RuntimeContext] = None, + executor: JobExecutor | None = None, + loading_context: LoadingContext | None = None, + runtime_context: RuntimeContext | None = None, ) -> None: if executor is None: executor = SingleJobExecutor() @@ -63,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: + def make(self, cwl: str | dict[str, Any]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 3c057ebbe..5694f6e11 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -4,7 +4,8 @@ http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html """ -from typing import Any, Callable, cast +from collections.abc import Callable +from typing import Any, cast def flatten(thing: Any) -> list[Any]: diff --git a/cwltool/job.py b/cwltool/job.py index df695aeb2..c46778b35 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -16,10 +16,10 @@ import time import uuid from abc import ABCMeta, abstractmethod -from collections.abc import Iterable, Mapping, MutableMapping, MutableSequence +from collections.abc import Callable, Iterable, Mapping, MutableMapping, MutableSequence from re import Match from threading import Timer -from typing import IO, TYPE_CHECKING, Callable, Optional, TextIO, Union, cast +from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil from prov.model import PROV @@ -99,7 +99,7 @@ def relink_initialworkdir( pass -def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Optional[Match[str]]: +def neverquote(string: str, pos: int = 0, endpos: int = 0) -> Match[str] | None: """No-op.""" return None @@ -118,9 +118,9 @@ def __init__( super().__init__() self.builder = builder self.joborder = joborder - self.stdin: Optional[str] = None - self.stderr: Optional[str] = None - self.stdout: Optional[str] = None + self.stdin: str | None = None + self.stderr: str | None = None + self.stdout: str | None = None self.successCodes: Iterable[int] = [] self.temporaryFailCodes: Iterable[int] = [] self.permanentFailCodes: Iterable[int] = [] @@ -130,11 +130,11 @@ def __init__( self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper - self.generatemapper: Optional[PathMapper] = None + self.generatemapper: PathMapper | None = None # set in CommandLineTool.job(i) self.collect_outputs = cast("CollectOutputsType", None) - self.output_callback: Optional[OutputCallbackType] = None + self.output_callback: OutputCallbackType | None = None self.outdir = "" self.tmpdir = "" @@ -144,13 +144,13 @@ def __init__( "listing": [], "basename": "", } - self.stagedir: Optional[str] = None + self.stagedir: str | None = None self.inplace_update = False - self.prov_obj: Optional[ProvenanceProfile] = None - self.parent_wf: Optional[ProvenanceProfile] = None - self.timelimit: Optional[int] = None + self.prov_obj: ProvenanceProfile | None = None + self.parent_wf: ProvenanceProfile | None = None + self.timelimit: int | None = None self.networkaccess: bool = False - self.mpi_procs: Optional[int] = None + self.mpi_procs: int | None = None def __repr__(self) -> str: """Represent this Job object.""" @@ -160,7 +160,7 @@ def __repr__(self) -> str: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: pass @@ -218,7 +218,7 @@ def _execute( runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, - monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, + monitor_function: Callable[["subprocess.Popen[str]"], None] | None = None, ) -> None: """Execute the tool, either directly or via script. @@ -293,8 +293,8 @@ def _execute( stdin_path = rmap[1] def stderr_stdout_log_path( - base_path_logs: str, stderr_or_stdout: Optional[str] - ) -> Optional[str]: + base_path_logs: str, stderr_or_stdout: str | None + ) -> str | None: if stderr_or_stdout is not None: abserr = os.path.join(base_path_logs, stderr_or_stdout) dnerr = os.path.dirname(abserr) @@ -316,8 +316,8 @@ def stderr_stdout_log_path( runtimeContext.secret_store.retrieve(cast(CWLOutputType, env)), ) - job_script_contents: Optional[str] = None - builder: Optional[Builder] = getattr(self, "builder", None) + job_script_contents: str | None = None + builder: Builder | None = getattr(self, "builder", None) if builder is not None: job_script_contents = builder.build_job_script(commands) rcode = _job_popen( @@ -464,7 +464,7 @@ def _required_env(self) -> dict[str, str]: """ def _preserve_environment_on_containers_warning( - self, varname: Optional[Iterable[str]] = None + self, varname: Iterable[str] | None = None ) -> None: """When running in a container, issue a warning.""" # By default, don't do anything; ContainerCommandLineJob below @@ -522,11 +522,11 @@ def process_monitor(self, sproc: "subprocess.Popen[str]") -> None: """Watch a process, logging its max memory usage.""" monitor = psutil.Process(sproc.pid) # Value must be list rather than integer to utilise pass-by-reference in python - memory_usage: MutableSequence[Optional[int]] = [None] + memory_usage: MutableSequence[int | None] = [None] mem_tm: "Optional[Timer]" = None - def get_tree_mem_usage(memory_usage: MutableSequence[Optional[int]]) -> None: + def get_tree_mem_usage(memory_usage: MutableSequence[int | None]) -> None: nonlocal mem_tm try: with monitor.oneshot(): @@ -565,7 +565,7 @@ class CommandLineJob(JobBase): def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if tmpdir_lock: with tmpdir_lock: @@ -627,7 +627,7 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: pass @abstractmethod @@ -635,7 +635,7 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> tuple[list[str], Optional[str]]: + ) -> tuple[list[str], str | None]: """Return the list of commands to run the selected container engine.""" @staticmethod @@ -645,7 +645,7 @@ def append_volume(runtime: list[str], source: str, target: str, writable: bool = @abstractmethod def add_file_or_directory_volume( - self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @@ -654,7 +654,7 @@ def add_writable_file_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable file mapping to the runtime option list.""" @@ -664,13 +664,13 @@ def add_writable_directory_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: """Append a writable directory mapping to the runtime option list.""" def _preserve_environment_on_containers_warning( - self, varnames: Optional[Iterable[str]] = None + self, varnames: Iterable[str] | None = None ) -> None: """When running in a container, issue a warning.""" if varnames is None: @@ -688,8 +688,8 @@ def create_file_and_add_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], - secret_store: Optional[SecretStore], + host_outdir_tgt: str | None, + secret_store: SecretStore | None, tmpdir_prefix: str, ) -> str: """Create the file and add a mapping.""" @@ -720,13 +720,13 @@ def add_volumes( pathmapper: PathMapper, runtime: list[str], tmpdir_prefix: str, - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, any_path_okay: bool = False, ) -> None: """Append volume mappings to the runtime option list.""" container_outdir = self.builder.outdir for key, vol in (itm for itm in pathmapper.items() if itm[1].staged): - host_outdir_tgt: Optional[str] = None + host_outdir_tgt: str | None = None if vol.target.startswith(container_outdir + "/"): host_outdir_tgt = os.path.join(self.outdir, vol.target[len(container_outdir) + 1 :]) if not host_outdir_tgt and not any_path_okay: @@ -735,22 +735,23 @@ def add_volumes( "the designated output directory, also know as " "$(runtime.outdir): {}".format(vol) ) - if vol.type in ("File", "Directory"): - self.add_file_or_directory_volume(runtime, vol, host_outdir_tgt) - elif vol.type == "WritableFile": - self.add_writable_file_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) - elif vol.type == "WritableDirectory": - self.add_writable_directory_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) - elif vol.type in ["CreateFile", "CreateWritableFile"]: - new_path = self.create_file_and_add_volume( - runtime, vol, host_outdir_tgt, secret_store, tmpdir_prefix - ) - pathmapper.update(key, new_path, vol.target, vol.type, vol.staged) + match vol.type: + case "File" | "Directory": + self.add_file_or_directory_volume(runtime, vol, host_outdir_tgt) + case "WritableFile": + self.add_writable_file_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) + case "WritableDirectory": + self.add_writable_directory_volume(runtime, vol, host_outdir_tgt, tmpdir_prefix) + case "CreateFile" | "CreateWritableFile": + new_path = self.create_file_and_add_volume( + runtime, vol, host_outdir_tgt, secret_store, tmpdir_prefix + ) + pathmapper.update(key, new_path, vol.target, vol.type, vol.staged) def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: debug = runtimeContext.debug if tmpdir_lock: @@ -763,7 +764,7 @@ def run( (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") self.prov_obj = runtimeContext.prov_obj - img_id = None + img_id: str | None = None user_space_docker_cmd = runtimeContext.user_space_docker_cmd if docker_req is not None and user_space_docker_cmd: # For user-space docker implementations, a local image name or ID @@ -868,7 +869,7 @@ def docker_monitor( # to stdout, but the container is frozen, thus allowing us to start the # monitoring process without dealing with the cidfile or too-fast # container execution - cid: Optional[str] = None + cid: str | None = None while cid is None: time.sleep(1) # This is needed to avoid a race condition where the job @@ -931,33 +932,29 @@ def docker_monitor( def _job_popen( commands: list[str], - stdin_path: Optional[str], - stdout_path: Optional[str], - stderr_path: Optional[str], + stdin_path: str | None, + stdout_path: str | None, + stderr_path: str | None, env: Mapping[str, str], cwd: str, make_job_dir: Callable[[], str], - job_script_contents: Optional[str] = None, - timelimit: Optional[int] = None, - name: Optional[str] = None, - monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, - default_stdout: Optional[Union[IO[bytes], TextIO]] = None, - default_stderr: Optional[Union[IO[bytes], TextIO]] = None, + job_script_contents: str | None = None, + timelimit: int | None = None, + name: str | None = None, + monitor_function: Callable[["subprocess.Popen[str]"], None] | None = None, + default_stdout: IO[bytes] | TextIO | None = None, + default_stderr: IO[bytes] | TextIO | None = None, ) -> int: if job_script_contents is None and not FORCE_SHELLED_POPEN: - stdin: Union[IO[bytes], int] = subprocess.PIPE + stdin: IO[bytes] | int = subprocess.PIPE if stdin_path is not None: stdin = open(stdin_path, "rb") - stdout = ( - default_stdout if default_stdout is not None else sys.stderr - ) # type: Union[IO[bytes], TextIO] + stdout: IO[bytes] | TextIO = default_stdout if default_stdout is not None else sys.stderr if stdout_path is not None: stdout = open(stdout_path, "wb") - stderr = ( - default_stderr if default_stderr is not None else sys.stderr - ) # type: Union[IO[bytes], TextIO] + stderr: IO[bytes] | TextIO = default_stderr if default_stderr is not None else sys.stderr if stderr_path is not None: stderr = open(stderr_path, "wb") @@ -980,7 +977,7 @@ def _job_popen( tm = None if timelimit is not None and timelimit > 0: - def terminate(): # type: () -> None + def terminate() -> None: try: _logger.warning( "[job %s] exceeded time limit of %d seconds and will be terminated", @@ -1056,7 +1053,7 @@ def terminate(): # type: () -> None tm = None if timelimit is not None and timelimit > 0: - def terminate(): # type: () -> None + def terminate() -> None: try: _logger.warning( "[job %s] exceeded time limit of %d seconds and will be terminated", diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index f7f4936cd..853c203be 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -9,7 +9,7 @@ import uuid from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -67,7 +67,7 @@ def default_loader( - fetcher_constructor: Optional[FetcherCallableType] = None, + fetcher_constructor: FetcherCallableType | None = None, enable_dev: bool = False, doc_cache: bool = True, ) -> Loader: @@ -81,11 +81,11 @@ def default_loader( def resolve_tool_uri( argsworkflow: str, - resolver: Optional[ResolverType] = None, - fetcher_constructor: Optional[FetcherCallableType] = None, - document_loader: Optional[Loader] = None, + resolver: ResolverType | None = None, + fetcher_constructor: FetcherCallableType | None = None, + document_loader: Loader | None = None, ) -> tuple[str, str]: - uri = None # type: Optional[str] + uri: str | None = None split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that if split.scheme and split.scheme in ["http", "https", "file"]: @@ -106,8 +106,8 @@ def resolve_tool_uri( def fetch_document( - argsworkflow: Union[str, CWLObjectType], - loadingContext: Optional[LoadingContext] = None, + argsworkflow: str | CWLObjectType, + loadingContext: LoadingContext | None = None, ) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: @@ -142,20 +142,20 @@ def fetch_document( def _convert_stdstreams_to_files( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str], + workflowobj: CWLObjectType | MutableSequence[CWLObjectType | str | int] | str, ) -> None: - if isinstance(workflowobj, MutableMapping): - if workflowobj.get("class") == "CommandLineTool": + match workflowobj: + case {"class": "CommandLineTool", **rest} if isinstance(workflowobj, MutableMapping): with SourceLine( workflowobj, "outputs", ValidationException, _logger.isEnabledFor(logging.DEBUG), ): - outputs = workflowobj.get("outputs", []) + outputs = rest.get("outputs", []) if not isinstance(outputs, CommentedSeq): raise ValidationException('"outputs" section is not ' "valid.") - for out in cast(MutableSequence[CWLObjectType], workflowobj.get("outputs", [])): + for out in cast(MutableSequence[CWLObjectType], outputs): if not isinstance(out, CommentedMap): raise ValidationException(f"Output {out!r} is not a valid OutputParameter.") for streamtype in ["stdout", "stderr"]: @@ -165,8 +165,8 @@ def _convert_stdstreams_to_files( "Not allowed to specify outputBinding when" " using %s shortcut." % streamtype ) - if streamtype in workflowobj: - filename = workflowobj[streamtype] + if streamtype in rest: + filename = rest[streamtype] else: filename = str( hashlib.sha1( # nosec @@ -176,13 +176,13 @@ def _convert_stdstreams_to_files( workflowobj[streamtype] = filename out["type"] = "File" out["outputBinding"] = cmap({"glob": filename}) - for inp in cast(MutableSequence[CWLObjectType], workflowobj.get("inputs", [])): + for inp in cast(MutableSequence[CWLObjectType], rest.get("inputs", [])): if inp.get("type") == "stdin": if "inputBinding" in inp: raise ValidationException( "Not allowed to specify inputBinding when" " using stdin shortcut." ) - if "stdin" in workflowobj: + if "stdin" in rest: raise ValidationException( "Not allowed to specify stdin path when" " using stdin type shortcut." ) @@ -192,7 +192,7 @@ def _convert_stdstreams_to_files( % cast(str, inp["id"]).rpartition("#")[2].split("/")[-1] ) inp["type"] = "File" - else: + case MutableMapping(): for entry in workflowobj.values(): _convert_stdstreams_to_files( cast( @@ -204,22 +204,22 @@ def _convert_stdstreams_to_files( entry, ) ) - if isinstance(workflowobj, MutableSequence): - for entry in workflowobj: - _convert_stdstreams_to_files( - cast( - Union[ - CWLObjectType, - MutableSequence[Union[CWLObjectType, str, int]], - str, - ], - entry, + case MutableSequence(): + for entry in workflowobj: + _convert_stdstreams_to_files( + cast( + Union[ + CWLObjectType, + MutableSequence[Union[CWLObjectType, str, int]], + str, + ], + entry, + ) ) - ) def _add_blank_ids( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]], + workflowobj: CWLObjectType | MutableSequence[CWLObjectType | str], ) -> None: if isinstance(workflowobj, MutableMapping): if ( @@ -247,20 +247,21 @@ def _add_blank_ids( def _fast_parser_convert_stdstreams_to_files( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], + processobj: cwl_v1_2.Process | MutableSequence[cwl_v1_2.Process], ) -> None: - if isinstance(processobj, cwl_v1_2.CommandLineTool): - cwl_v1_2_utils.convert_stdstreams_to_files(processobj) - elif isinstance(processobj, cwl_v1_2.Workflow): - for st in processobj.steps: - _fast_parser_convert_stdstreams_to_files(st.run) - elif isinstance(processobj, MutableSequence): - for p in processobj: - _fast_parser_convert_stdstreams_to_files(p) + match processobj: + case cwl_v1_2.CommandLineTool(): + cwl_v1_2_utils.convert_stdstreams_to_files(processobj) + case cwl_v1_2.Workflow(steps=steps): + for st in steps: + _fast_parser_convert_stdstreams_to_files(st.run) + case MutableSequence(): + for p in processobj: + _fast_parser_convert_stdstreams_to_files(p) def _fast_parser_expand_hint_class( - hints: Optional[Any], loadingOptions: cwl_v1_2.LoadingOptions + hints: Any | None, loadingOptions: cwl_v1_2.LoadingOptions ) -> None: if isinstance(hints, MutableSequence): for h in hints: @@ -271,19 +272,19 @@ def _fast_parser_expand_hint_class( def _fast_parser_handle_hints( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], + processobj: cwl_v1_2.Process | MutableSequence[cwl_v1_2.Process], loadingOptions: cwl_v1_2.LoadingOptions, ) -> None: if isinstance(processobj, (cwl_v1_2.CommandLineTool, cwl_v1_2.Workflow)): _fast_parser_expand_hint_class(processobj.hints, loadingOptions) - - if isinstance(processobj, cwl_v1_2.Workflow): - for st in processobj.steps: - _fast_parser_expand_hint_class(st.hints, loadingOptions) - _fast_parser_handle_hints(st.run, loadingOptions) - elif isinstance(processobj, MutableSequence): - for p in processobj: - _fast_parser_handle_hints(p, loadingOptions) + match processobj: + case cwl_v1_2.Workflow(steps=steps): + for st in steps: + _fast_parser_expand_hint_class(st.hints, loadingOptions) + _fast_parser_handle_hints(st.run, loadingOptions) + case MutableSequence(): + for p in processobj: + _fast_parser_handle_hints(p, loadingOptions) def update_index(document_loader: Loader, pr: CommentedMap) -> None: @@ -292,12 +293,12 @@ def update_index(document_loader: Loader, pr: CommentedMap) -> None: def fast_parser( - workflowobj: Union[CommentedMap, CommentedSeq, None], - fileuri: Optional[str], + workflowobj: CommentedMap | CommentedSeq | None, + fileuri: str | None, uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[CommentedMap | CommentedSeq, CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -313,7 +314,7 @@ def fast_parser( _fast_parser_convert_stdstreams_to_files(objects) _fast_parser_handle_hints(objects, loadopt) - processobj: Union[MutableMapping[str, Any], MutableSequence[Any], float, str, None] + processobj: MutableMapping[str, Any] | MutableSequence[Any] | float | str | None processobj = cwl_v1_2.save(objects, relative_uris=False) @@ -367,7 +368,7 @@ def fast_parser( def resolve_and_validate_document( loadingContext: LoadingContext, - workflowobj: Union[CommentedMap, CommentedSeq], + workflowobj: CommentedMap | CommentedSeq, uri: str, preprocess_only: bool = False, ) -> tuple[LoadingContext, str]: @@ -431,24 +432,25 @@ def resolve_and_validate_document( "\n{}".format("\n".join(versions)) ) - if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj: - loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) - del jobobj["http://commonwl.org/cwltool#overrides"] - - if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj: - if cwlVersion not in ("v1.1.0-dev1", "v1.1"): - raise ValidationException( - "`cwl:requirements` in the input object is not part of CWL " - "v1.0. You can adjust to use `cwltool:overrides` instead; or you " - "can set the cwlVersion to v1.1 or greater." + if isinstance(jobobj, CommentedMap): + if "http://commonwl.org/cwltool#overrides" in jobobj: + loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri)) + del jobobj["http://commonwl.org/cwltool#overrides"] + + if "https://w3id.org/cwl/cwl#requirements" in jobobj: + if cwlVersion not in ("v1.1.0-dev1", "v1.1"): + raise ValidationException( + "`cwl:requirements` in the input object is not part of CWL " + "v1.0. You can adjust to use `cwltool:overrides` instead; or you " + "can set the cwlVersion to v1.1 or greater." + ) + loadingContext.overrides_list.append( + { + "overrideTarget": uri, + "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], + } ) - loadingContext.overrides_list.append( - { - "overrideTarget": uri, - "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"], - } - ) - del jobobj["https://w3id.org/cwl/cwl#requirements"] + del jobobj["https://w3id.org/cwl/cwl#requirements"] (sch_document_loader, avsc_names) = process.get_schema(cwlVersion)[:2] @@ -551,14 +553,12 @@ def resolve_and_validate_document( return loadingContext, uri -def make_tool( - uri: Union[str, CommentedMap, CommentedSeq], loadingContext: LoadingContext -) -> Process: +def make_tool(uri: str | CommentedMap | CommentedSeq, loadingContext: LoadingContext) -> Process: """Make a Python CWL object.""" if loadingContext.loader is None: raise ValueError("loadingContext must have a loader") - resolveduri: Union[float, str, CommentedMap, CommentedSeq, None] + resolveduri: float | str | CommentedMap | CommentedSeq | None metadata: CWLObjectType if loadingContext.fast_parser and isinstance(uri, str) and not loadingContext.skip_resolve_all: @@ -569,21 +569,24 @@ def make_tool( resolveduri, metadata = loadingContext.loader.resolve_ref(uri) processobj = None - if isinstance(resolveduri, MutableSequence): - for obj in resolveduri: - if obj["id"].endswith("#main"): - processobj = obj - break - if not processobj: - raise GraphTargetMissingException( - "Tool file contains graph of multiple objects, must specify " - "one of #%s" - % ", #".join(urllib.parse.urldefrag(i["id"])[1] for i in resolveduri if "id" in i) - ) - elif isinstance(resolveduri, MutableMapping): - processobj = resolveduri - else: - raise Exception("Must resolve to list or dict") + match resolveduri: + case MutableSequence(): + for obj in resolveduri: + if obj["id"].endswith("#main"): + processobj = obj + break + if not processobj: + raise GraphTargetMissingException( + "Tool file contains graph of multiple objects, must specify " + "one of #%s" + % ", #".join( + urllib.parse.urldefrag(i["id"])[1] for i in resolveduri if "id" in i + ) + ) + case MutableMapping(): + processobj = resolveduri + case _: + raise Exception(f"Must resolve to list or dict: {resolveduri}") tool = loadingContext.construct_tool_object(processobj, loadingContext) @@ -597,8 +600,8 @@ def make_tool( def load_tool( - argsworkflow: Union[str, CWLObjectType], - loadingContext: Optional[LoadingContext] = None, + argsworkflow: str | CWLObjectType, + loadingContext: LoadingContext | None = None, ) -> Process: loadingContext, workflowobj, uri = fetch_document(argsworkflow, loadingContext) @@ -633,7 +636,7 @@ def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: def recursive_resolve_and_validate_document( loadingContext: LoadingContext, - workflowobj: Union[CommentedMap, CommentedSeq], + workflowobj: CommentedMap | CommentedSeq, uri: str, preprocess_only: bool = False, ) -> tuple[LoadingContext, str, Process]: diff --git a/cwltool/main.py b/cwltool/main.py index 41a5b03f6..4cbf356c7 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,9 +15,9 @@ import urllib import warnings from codecs import getwriter -from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from collections.abc import Callable, Mapping, MutableMapping, MutableSequence, Sized from importlib.resources import files -from typing import IO, Any, Callable, Optional, Union, cast +from typing import IO, Any, Union, cast import argcomplete import coloredlogs @@ -166,8 +166,8 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( - inptype: Optional[CWLOutputType], - default: Optional[CWLOutputType], + inptype: CWLOutputType | None, + default: CWLOutputType | None, ) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None @@ -186,39 +186,39 @@ def generate_example_input( [("class", "Directory"), ("path", "a/directory/path")] ), } - if isinstance(inptype, MutableSequence): - optional = False - if "null" in inptype: - inptype.remove("null") - optional = True - if len(inptype) == 1: - example, comment = generate_example_input(inptype[0], default) - if optional: - if comment: + match inptype: + case MutableSequence(): + optional = False + if "null" in inptype: + inptype.remove("null") + optional = True + if len(inptype) == 1: + example, comment = generate_example_input(inptype[0], default) + if optional: + if comment: + comment = f"{comment} (optional)" + else: + comment = "optional" + else: + example, comment = generate_example_input(inptype[0], default) + type_names = [] + for entry in inptype: + value, e_comment = generate_example_input(entry, default) + if e_comment: + type_names.append(e_comment) + comment = "one of " + ", ".join(type_names) + if optional: comment = f"{comment} (optional)" - else: - comment = "optional" - else: - example, comment = generate_example_input(inptype[0], default) - type_names = [] - for entry in inptype: - value, e_comment = generate_example_input(entry, default) - if e_comment: - type_names.append(e_comment) - comment = "one of " + ", ".join(type_names) - if optional: - comment = f"{comment} (optional)" - elif isinstance(inptype, Mapping) and "type" in inptype: - if inptype["type"] == "array": - first_item = cast(MutableSequence[CWLObjectType], inptype["items"])[0] - items_len = len(cast(Sized, inptype["items"])) + case {"type": "array", "items": list(items)}: + first_item = cast(CWLObjectType, items[0]) + items_len = len(items) if items_len == 1 and "type" in first_item and first_item["type"] == "enum": # array of just an enum then list all the options example = first_item["symbols"] if "name" in first_item: comment = 'array of type "{}".'.format(first_item["name"]) else: - value, comment = generate_example_input(inptype["items"], None) + value, comment = generate_example_input(items, None) comment = "array of " + comment if items_len == 1: example = [value] @@ -226,46 +226,44 @@ def generate_example_input( example = value if default is not None: example = default - elif inptype["type"] == "enum": - symbols = cast(list[str], inptype["symbols"]) + case {"type": "enum", "symbols": list(symbols), **rest}: if default is not None: example = default - elif "default" in inptype: - example = inptype["default"] - elif len(cast(Sized, inptype["symbols"])) == 1: + elif "default" in rest: + example = rest["default"] + elif len(symbols) == 1: example = symbols[0] else: - example = "{}_enum_value".format(inptype.get("name", "valid")) + example = "{}_enum_value".format(rest.get("name", "valid")) comment = 'enum; valid values: "{}"'.format('", "'.join(symbols)) - elif inptype["type"] == "record": + case {"type": "record", "fields": list(fields), **rest}: example = ruamel.yaml.comments.CommentedMap() - if "name" in inptype: - comment = '"{}" record type.'.format(inptype["name"]) + if "name" in rest: + comment = '"{}" record type.'.format(rest["name"]) else: comment = "Anonymous record type." - for field in cast(list[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], fields): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) - elif "default" in inptype: - example = inptype["default"] - comment = f"default value of type {inptype['type']!r}" - else: - example = defaults.get(cast(str, inptype["type"]), str(inptype)) - comment = f"type {inptype['type']!r}" - else: - if not default: + case {"type": str(inp_type), "default": default}: + example = default + comment = f"default value of type {inp_type!r}" + case {"type": str(inp_type)}: + example = defaults.get(inp_type, str(inptype)) + comment = f"type {inp_type!r}" + case object() if not default: example = defaults.get(str(inptype), str(inptype)) comment = f"type {inptype!r}" - else: + case _: example = default comment = f"default value of type {inptype!r}." return example, comment def realize_input_schema( - input_types: MutableSequence[Union[str, CWLObjectType]], + input_types: MutableSequence[str | CWLObjectType], schema_defs: MutableMapping[str, CWLObjectType], -) -> MutableSequence[Union[str, CWLObjectType]]: +) -> MutableSequence[str | CWLObjectType]: """Replace references to named typed with the actual types.""" for index, entry in enumerate(input_types): if isinstance(entry, str): @@ -338,10 +336,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: def load_job_order( args: argparse.Namespace, stdin: IO[Any], - fetcher_constructor: Optional[FetcherCallableType], + fetcher_constructor: FetcherCallableType | None, overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[CWLObjectType | None, str, Loader]: job_order_object = None job_order_file = None @@ -394,7 +392,7 @@ def load_job_order( def init_job_order( - job_order_object: Optional[CWLObjectType], + job_order_object: CWLObjectType | None, args: argparse.Namespace, process: Process, loader: Loader, @@ -403,9 +401,9 @@ def init_job_order( relative_deps: str = "primary", make_fs_access: Callable[[str], StdFsAccess] = StdFsAccess, input_basedir: str = "", - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, input_required: bool = True, - runtime_context: Optional[RuntimeContext] = None, + runtime_context: RuntimeContext | None = None, ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: @@ -503,7 +501,7 @@ def expand_formats(p: CWLObjectType) -> None: builder.bind_input( process.inputs_record_schema, job_order_object, discover_secondaryFiles=True ) - basedir: Optional[str] = None + basedir: str | None = None uri = cast(str, job_order_object[jobloader_id_name]) if uri == args.workflow: basedir = os.path.dirname(uri) @@ -549,7 +547,7 @@ def printdeps( stdout: IO[str], relative_deps: str, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, nestdirs: bool = True, ) -> None: """Print a JSON representation of the dependencies of the CWL document.""" @@ -566,7 +564,7 @@ def prov_deps( obj: CWLObjectType, document_loader: Loader, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, ) -> CWLObjectType: deps = find_deps(obj, document_loader, uri, basedir=basedir) @@ -587,7 +585,7 @@ def find_deps( obj: CWLObjectType, document_loader: Loader, uri: str, - basedir: Optional[str] = None, + basedir: str | None = None, nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" @@ -597,7 +595,7 @@ def find_deps( "format": CWL_IANA, } - def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: + def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) sfs = scandeps( @@ -639,7 +637,7 @@ def supported_cwl_versions(enable_dev: bool) -> list[str]: def setup_schema( - args: argparse.Namespace, custom_schema_callback: Optional[Callable[[], None]] + args: argparse.Namespace, custom_schema_callback: Callable[[], None] | None ) -> None: if custom_schema_callback is not None: custom_schema_callback() @@ -669,7 +667,7 @@ def __init__(self) -> None: """Use the default formatter with our custom formatstring.""" super().__init__("[%(asctime)sZ] %(message)s") - def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) -> str: + def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: """Override the default formatTime to include the timezone.""" formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(float(record.created))) with_msecs = f"{formatted_time},{record.msecs:03f}" @@ -682,7 +680,7 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[list[str]] = None, + argsl: list[str] | None = None, ) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") @@ -708,7 +706,7 @@ def setup_provenance( def setup_loadingContext( - loadingContext: Optional[LoadingContext], + loadingContext: LoadingContext | None, runtimeContext: RuntimeContext, args: argparse.Namespace, ) -> LoadingContext: @@ -783,7 +781,7 @@ def choose_target( args: argparse.Namespace, tool: Process, loading_context: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the Workflow, extract the subset matches all the args.targets.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") @@ -819,7 +817,7 @@ def choose_step( args: argparse.Namespace, tool: Process, loading_context: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the given Workflow and extract just args.single_step.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") @@ -851,7 +849,7 @@ def choose_process( args: argparse.Namespace, tool: Process, loadingContext: LoadingContext, -) -> Optional[Process]: +) -> Process | None: """Walk the given Workflow and extract just args.single_process.""" if loadingContext.loader is None: raise Exception("loadingContext.loader cannot be None") @@ -883,7 +881,7 @@ def choose_process( def check_working_directories( runtimeContext: RuntimeContext, -) -> Optional[int]: +) -> int | None: """Make any needed working directories.""" for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if ( @@ -930,7 +928,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, dict[str, Any]] = t["run"] + run: str | Process | dict[str, Any] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -941,18 +939,18 @@ def print_targets( def main( - argsl: Optional[list[str]] = None, - args: Optional[argparse.Namespace] = None, - job_order_object: Optional[CWLObjectType] = None, + argsl: list[str] | None = None, + args: argparse.Namespace | None = None, + job_order_object: CWLObjectType | None = None, stdin: IO[Any] = sys.stdin, - stdout: Optional[IO[str]] = None, + stdout: IO[str] | None = None, stderr: IO[Any] = sys.stderr, versionfunc: Callable[[], str] = versionstring, - logger_handler: Optional[logging.Handler] = None, - custom_schema_callback: Optional[Callable[[], None]] = None, - executor: Optional[JobExecutor] = None, - loadingContext: Optional[LoadingContext] = None, - runtimeContext: Optional[RuntimeContext] = None, + logger_handler: logging.Handler | None = None, + custom_schema_callback: Callable[[], None] | None = None, + executor: JobExecutor | None = None, + loadingContext: LoadingContext | None = None, + runtimeContext: RuntimeContext | None = None, input_required: bool = True, ) -> int: if stdout is None: # force UTF-8 even if the console is configured differently @@ -970,7 +968,7 @@ def main( _logger.removeHandler(defaultStreamHandler) workflowobj = None - prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None + prov_log_handler: logging.StreamHandler[ProvOut] | None = None global docker_exe user_agent = "cwltool" @@ -1054,7 +1052,7 @@ def main( setup_schema(args, custom_schema_callback) - prov_log_stream: Optional[Union[io.TextIOWrapper, WritableBagFile]] = None + prov_log_stream: io.TextIOWrapper | WritableBagFile | None = None if args.provenance: try: prov_log_stream, prov_log_handler = setup_provenance(args, runtimeContext, argsl) @@ -1430,10 +1428,10 @@ def loc_to_path(obj: CWLObjectType) -> None: def find_default_container( builder: HasReqsHints, - default_container: Optional[str] = None, - use_biocontainers: Optional[bool] = None, - container_image_cache_path: Optional[str] = None, -) -> Optional[str]: + default_container: str | None = None, + use_biocontainers: bool | None = None, + container_image_cache_path: str | None = None, +) -> str | None: """Find a container.""" if not default_container and use_biocontainers: from .software_requirements import get_container_from_software_requirements diff --git a/cwltool/mpi.py b/cwltool/mpi.py index a7bdcbe03..cea53e2f7 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -4,7 +4,7 @@ import os import re from collections.abc import Mapping, MutableMapping -from typing import Optional, TypeVar, Union +from typing import TypeVar from schema_salad.utils import yaml_no_ts @@ -18,11 +18,11 @@ def __init__( self, runner: str = "mpirun", nproc_flag: str = "-n", - default_nproc: Union[int, str] = 1, - extra_flags: Optional[list[str]] = None, - env_pass: Optional[list[str]] = None, - env_pass_regex: Optional[list[str]] = None, - env_set: Optional[Mapping[str, str]] = None, + default_nproc: int | str = 1, + extra_flags: list[str] | None = None, + env_pass: list[str] | None = None, + env_pass_regex: list[str] | None = None, + env_set: Mapping[str, str] | None = None, ) -> None: """ Initialize from the argument mapping. diff --git a/cwltool/pack.py b/cwltool/pack.py index 6d96c83a1..8c250c51b 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,8 +2,8 @@ import copy import urllib -from collections.abc import MutableMapping, MutableSequence -from typing import Any, Callable, Optional, Union, cast +from collections.abc import Callable, MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -19,7 +19,7 @@ def find_run( - d: Union[CWLObjectType, ResolveType], + d: CWLObjectType | ResolveType, loadref: LoadRefType, runs: set[str], ) -> None: @@ -36,7 +36,7 @@ def find_run( def find_ids( - d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], + d: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType] | None, ids: set[str], ) -> None: if isinstance(d, MutableSequence): @@ -79,7 +79,7 @@ def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> No def import_embed( - d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], + d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType, seen: set[str], ) -> None: if isinstance(d, MutableSequence): @@ -106,8 +106,8 @@ def import_embed( def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[dict[str, str]] = None, - loader: Optional[Loader] = None, + rewrite_out: dict[str, str] | None = None, + loader: Loader | None = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been # updated to the internal CWL version. We need to reload the @@ -147,7 +147,7 @@ def pack( found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} - def loadref(base: Optional[str], lr_uri: str) -> ResolveType: + def loadref(base: str | None, lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() lr_loadingContext.metadata = {} lr_loadingContext, lr_workflowobj, lr_uri = fetch_document(lr_uri, lr_loadingContext) diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index d09641fd8..0cf5f7086 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -26,12 +26,12 @@ class MapperEnt(NamedTuple): """ target: str """The path on the target file system (under stagedir)""" - type: Optional[str] + type: str | None """ The object type. One of "File", "Directory", "CreateFile", "WritableFile", or "CreateWritableFile". """ - staged: Optional[bool] + staged: bool | None """If the File has been staged yet.""" @@ -232,7 +232,7 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[tuple[str, str]]: + ) -> tuple[str, str] | None: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: @@ -240,7 +240,7 @@ def reversemap( return None def update( - self, key: str, resolved: str, target: str, ctype: Optional[str], stage: Optional[bool] + self, key: str, resolved: str, target: str, ctype: str | None, stage: bool | None ) -> MapperEnt: """Update an existing entry.""" m = MapperEnt(resolved, target, ctype, stage) diff --git a/cwltool/process.py b/cwltool/process.py index ce0f52b73..75069b859 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,10 +13,17 @@ import textwrap import urllib.parse import uuid -from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized +from collections.abc import ( + Callable, + Iterable, + Iterator, + MutableMapping, + MutableSequence, + Sized, +) from importlib.resources import files from os import scandir -from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -145,12 +152,10 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: dict[ - str, tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] -] = {} -SCHEMA_FILE: Optional[CWLObjectType] = None -SCHEMA_DIR: Optional[CWLObjectType] = None -SCHEMA_ANY: Optional[CWLObjectType] = None +SCHEMA_CACHE: dict[str, tuple[Loader, Names | SchemaParseException, CWLObjectType, Loader]] = {} +SCHEMA_FILE: CWLObjectType | None = None +SCHEMA_DIR: CWLObjectType | None = None +SCHEMA_ANY: CWLObjectType | None = None custom_schemas: dict[str, tuple[str, str]] = {} @@ -170,11 +175,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Names | SchemaParseException, CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, str | Graph | bool] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -216,10 +221,10 @@ def shortname(inputid: str) -> str: def stage_files( pathmapper: PathMapper, - stage_func: Optional[Callable[[str, str], None]] = None, + stage_func: Callable[[str, str], None] | None = None, ignore_writable: bool = False, symlink: bool = True, - secret_store: Optional[SecretStore] = None, + secret_store: SecretStore | None = None, fix_conflicts: bool = False, ) -> None: """ @@ -257,37 +262,35 @@ def stage_files( continue if not os.path.exists(os.path.dirname(entry.target)): os.makedirs(os.path.dirname(entry.target)) - if entry.type in ("File", "Directory") and os.path.exists(entry.resolved): - if symlink: # Use symlink func if allowed - os.symlink(entry.resolved, entry.target) - elif stage_func is not None: + match entry.type: + case "File" | "Directory" if os.path.exists(entry.resolved) and symlink: + os.symlink(entry.resolved, entry.target) # Use symlink func if allowed + case "File" | "Directory" if os.path.exists(entry.resolved) and stage_func is not None: stage_func(entry.resolved, entry.target) - elif ( - entry.type == "Directory" - and not os.path.exists(entry.target) - and entry.resolved.startswith("_:") - ): - os.makedirs(entry.target) - elif entry.type == "WritableFile" and not ignore_writable: - shutil.copy(entry.resolved, entry.target) - ensure_writable(entry.target) - elif entry.type == "WritableDirectory" and not ignore_writable: - if entry.resolved.startswith("_:"): + case "Directory" if not os.path.exists(entry.target) and entry.resolved.startswith( + "_:" + ): os.makedirs(entry.target) - else: - shutil.copytree(entry.resolved, entry.target) - ensure_writable(entry.target, include_root=True) - elif entry.type == "CreateFile" or entry.type == "CreateWritableFile": - with open(entry.target, "w") as new: - if secret_store is not None: - new.write(cast(str, secret_store.retrieve(entry.resolved))) - else: - new.write(entry.resolved) - if entry.type == "CreateFile": - os.chmod(entry.target, stat.S_IRUSR) # Read only - else: # it is a "CreateWritableFile" + case "WritableFile" if not ignore_writable: + shutil.copy(entry.resolved, entry.target) ensure_writable(entry.target) - pathmapper.update(key, entry.target, entry.target, entry.type, entry.staged) + case "WritableDirectory" if not ignore_writable: + if entry.resolved.startswith("_:"): + os.makedirs(entry.target) + else: + shutil.copytree(entry.resolved, entry.target) + ensure_writable(entry.target, include_root=True) + case "CreateFile" | "CreateWritableFile" as etype: + with open(entry.target, "w") as new: + if secret_store is not None: + new.write(cast(str, secret_store.retrieve(entry.resolved))) + else: + new.write(entry.resolved) + if etype == "CreateFile": + os.chmod(entry.target, stat.S_IRUSR) # Read only + else: # it is a "CreateWritableFile" + ensure_writable(entry.target) + pathmapper.update(key, entry.target, entry.target, entry.type, entry.staged) def relocateOutputs( @@ -305,7 +308,7 @@ def relocateOutputs( return outputObj def _collectDirEntries( - obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None], + obj: CWLObjectType | MutableSequence[CWLObjectType] | None, ) -> Iterator[CWLObjectType]: if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): @@ -424,31 +427,31 @@ def fill_in_defaults( def avroize_type( - field_type: Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None], + field_type: CWLObjectType | MutableSequence[Any] | CWLOutputType | None, name_prefix: str = "", -) -> Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None]: +) -> CWLObjectType | MutableSequence[Any] | CWLOutputType | None: """Add missing information to a type so that CWL types are valid.""" - if isinstance(field_type, MutableSequence): - for i, field in enumerate(field_type): - field_type[i] = avroize_type(field, name_prefix) - elif isinstance(field_type, MutableMapping): - if field_type["type"] in ("enum", "record"): - if "name" not in field_type: - field_type["name"] = name_prefix + str(uuid.uuid4()) - if field_type["type"] == "record": - field_type["fields"] = avroize_type( - cast(MutableSequence[CWLOutputType], field_type["fields"]), name_prefix - ) - elif field_type["type"] == "array": - field_type["items"] = avroize_type( - cast(MutableSequence[CWLOutputType], field_type["items"]), name_prefix + match field_type: + case MutableSequence(): + for i, field in enumerate(field_type): + field_type[i] = avroize_type(field, name_prefix) + case {"type": "enum" | "record" as f_type, **rest}: + if "name" not in rest: + cast(CWLObjectType, field_type)["name"] = name_prefix + str(uuid.uuid4()) + if f_type == "record": + cast(CWLObjectType, field_type)["fields"] = avroize_type( + cast(MutableSequence[CWLOutputType], rest["fields"]), name_prefix + ) + case {"type": "array", "items": items}: + cast(CWLObjectType, field_type)["items"] = avroize_type( + cast(MutableSequence[CWLOutputType], items), name_prefix ) - else: - field_type["type"] = avroize_type(field_type["type"], name_prefix) - elif field_type == "File": - return "org.w3id.cwl.cwl.File" - elif field_type == "Directory": - return "org.w3id.cwl.cwl.Directory" + case {"type": f_type}: + cast(CWLObjectType, field_type)["type"] = avroize_type(f_type, name_prefix) + case "File": + return "org.w3id.cwl.cwl.File" + case "Directory": + return "org.w3id.cwl.cwl.Directory" return field_type @@ -475,8 +478,8 @@ def get_overrides(overrides: MutableSequence[CWLObjectType], toolid: str) -> CWL def var_spool_cwl_detector( obj: CWLOutputType, - item: Optional[Any] = None, - obj_key: Optional[Any] = None, + item: Any | None = None, + obj_key: Any | None = None, ) -> bool: """Detect any textual reference to /var/spool/cwl.""" r = False @@ -497,23 +500,27 @@ def var_spool_cwl_detector( return r -def eval_resource( - builder: Builder, resource_req: Union[str, int, float] -) -> Optional[Union[str, int, float]]: +def eval_resource(builder: Builder, resource_req: str | int | float) -> str | int | float | None: + """Evaluate any CWL expressions inside a ResourceRequirement.""" if isinstance(resource_req, str) and expression.needs_parsing(resource_req): result = builder.do_eval(resource_req) - if isinstance(result, float): - if ORDERED_VERSIONS.index(builder.cwlVersion) >= ORDERED_VERSIONS.index("v1.2.0-dev4"): + match result: + case float(f_result) if ORDERED_VERSIONS.index( + builder.cwlVersion + ) >= ORDERED_VERSIONS.index("v1.2.0-dev4"): + return f_result + case float(): + raise WorkflowException( + "Floats are not valid in resource requirement expressions prior " + f"to CWL v1.2: {resource_req} returned {result}." + ) + case str() | int() | None: return result - raise WorkflowException( - "Floats are not valid in resource requirement expressions prior " - f"to CWL v1.2: {resource_req} returned {result}." - ) - if isinstance(result, (str, int)) or result is None: - return result - raise WorkflowException( - f"Got incorrect return type {type(result)} from resource expression evaluation of {resource_req}." - ) + case _: + raise WorkflowException( + f"Got incorrect return type {type(result)} from resource " + "expression evaluation of {resource_req}." + ) return resource_req @@ -581,7 +588,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.doc_loader = loadingContext.loader self.doc_schema = loadingContext.avsc_names - self.formatgraph: Optional[Graph] = None + self.formatgraph: Graph | None = None if self.doc_loader is not None: self.formatgraph = self.doc_loader.graph @@ -665,7 +672,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[dict[str, Union[list[str], str, int]]] = None + validate_js_options: dict[str, list[str] | str | int] | None = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -966,7 +973,7 @@ def inc(d: list[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> dict[str, Union[int, float]]: + ) -> dict[str, int | float]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -976,7 +983,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: dict[str, Union[int, float, str]] = { + request: dict[str, int | float | str] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -1001,8 +1008,8 @@ def evalResources( ): if rsc is None: continue - mn: Optional[Union[int, float]] = None - mx: Optional[Union[int, float]] = None + mn: int | float | None = None + mx: int | float | None = None if rsc.get(a + "Min"): with SourceLine(rsc, f"{a}Min", WorkflowException, runtimeContext.debug): mn = cast( @@ -1041,7 +1048,7 @@ def evalResources( def checkRequirements( self, - rec: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType, None], + rec: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType | None, supported_process_requirements: Iterable[str], ) -> None: """Check the presence of unsupported requirements.""" @@ -1107,7 +1114,7 @@ def __str__(self) -> str: _names: set[str] = set() -def uniquename(stem: str, names: Optional[set[str]] = None) -> str: +def uniquename(stem: str, names: set[str] | None = None) -> str: """Construct a thread-unique name using the given stem as a prefix.""" if names is None: names = _names @@ -1175,10 +1182,10 @@ def mergedirs( def scandeps( base: str, - doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], + doc: CWLObjectType | MutableSequence[CWLObjectType], reffields: set[str], urlfields: set[str], - loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], + loadref: Callable[[str, str], CommentedMap | CommentedSeq | str | None], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, ) -> MutableSequence[CWLObjectType]: @@ -1219,52 +1226,54 @@ def scandeps( } if "basename" in doc: deps["basename"] = doc["basename"] - if doc["class"] == "Directory" and "listing" in doc: - deps["listing"] = doc["listing"] - if doc["class"] == "File" and "secondaryFiles" in doc: - deps["secondaryFiles"] = cast( - CWLOutputType, - scandeps( - base, - cast( - Union[CWLObjectType, MutableSequence[CWLObjectType]], - doc["secondaryFiles"], + match doc: + case {"class": "Directory", "listing": listing}: + deps["listing"] = listing + case {"class": "File", "secondaryFiles": sec_files}: + deps["secondaryFiles"] = cast( + CWLOutputType, + scandeps( + base, + cast( + Union[CWLObjectType, MutableSequence[CWLObjectType]], + sec_files, + ), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ), - ) + ) if nestdirs: deps = nestdir(base, deps) r.append(deps) else: - if doc["class"] == "Directory" and "listing" in doc: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], doc["listing"]), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, + match doc: + case {"class": "Directory", "listing": listing}: + r.extend( + scandeps( + base, + cast(MutableSequence[CWLObjectType], listing), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) ) - ) - elif doc["class"] == "File" and "secondaryFiles" in doc: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], doc["secondaryFiles"]), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, + case {"class": "File", "secondaryFiles": sec_files}: + r.extend( + scandeps( + base, + cast(MutableSequence[CWLObjectType], sec_files), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) ) - ) for k, v in doc.items(): if k in reffields: diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 07123f906..eb7eca076 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Optional, cast +from typing import cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -19,10 +19,10 @@ class ProcessGeneratorJob: def __init__(self, procgenerator: "ProcessGenerator") -> None: """Create a ProccessGenerator Job.""" self.procgenerator = procgenerator - self.jobout = None # type: Optional[CWLObjectType] - self.processStatus = None # type: Optional[str] + self.jobout: CWLObjectType | None = None + self.processStatus: str | None = None - def receive_output(self, jobout: Optional[CWLObjectType], processStatus: str) -> None: + def receive_output(self, jobout: CWLObjectType | None, processStatus: str) -> None: """Process the results.""" self.jobout = jobout self.processStatus = processStatus @@ -69,12 +69,12 @@ def __init__( ) -> None: """Create a ProcessGenerator from the given dictionary and context.""" super().__init__(toolpath_object, loadingContext) - self.loadingContext = loadingContext # type: LoadingContext + self.loadingContext: LoadingContext = loadingContext try: if isinstance(toolpath_object["run"], CommentedMap): - self.embedded_tool = loadingContext.construct_tool_object( + self.embedded_tool: Process = loadingContext.construct_tool_object( toolpath_object["run"], loadingContext - ) # type: Process + ) else: loadingContext.metadata = {} self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index 83586e1da..ed8c1b410 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -3,14 +3,13 @@ import os import urllib from pathlib import Path -from typing import Optional from schema_salad.ref_resolver import Loader from .loghandler import _logger -def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: +def resolve_local(document_loader: Loader | None, uri: str) -> str | None: """Use the local resolver to find the target of the URI.""" pathpart, frag = urllib.parse.urldefrag(uri) @@ -41,7 +40,7 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: return None -def tool_resolver(document_loader: Loader, uri: str) -> Optional[str]: +def tool_resolver(document_loader: Loader, uri: str) -> str | None: """Try both the local resolver and the GA4GH TRS resolver, in that order.""" for r in [resolve_local, resolve_ga4gh_tool]: ret = r(document_loader, uri) @@ -64,7 +63,7 @@ def tool_resolver(document_loader: Loader, uri: str) -> Optional[str]: GA4GH_TRS_PRIMARY_DESCRIPTOR = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor/{3}" -def resolve_ga4gh_tool(document_loader: Loader, uri: str) -> Optional[str]: +def resolve_ga4gh_tool(document_loader: Loader, uri: str) -> str | None: """Use the GA4GH TRS API to resolve a tool reference.""" path, version = uri.partition(":")[::2] if not version: diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 5a81ce20c..d2cb42e99 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,7 +4,7 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Optional, TextIO, Union +from typing import BinaryIO, TextIO def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: @@ -55,20 +55,20 @@ def main(argv: list[str]) -> int: stdout_path = popen_description["stdout_path"] stderr_path = popen_description["stderr_path"] if stdin_path is not None: - stdin: Union[BinaryIO, int] = open(stdin_path, "rb") + stdin: BinaryIO | int = open(stdin_path, "rb") else: stdin = subprocess.PIPE if stdout_path is not None: - stdout: Union[BinaryIO, TextIO] = open(stdout_path, "wb") + stdout: BinaryIO | TextIO = open(stdout_path, "wb") else: stdout = sys.stderr if stderr_path is not None: - stderr: Union[BinaryIO, TextIO] = open(stderr_path, "wb") + stderr: BinaryIO | TextIO = open(stderr_path, "wb") else: stderr = sys.stderr try: - env_script: Optional[str] = argv[2] + env_script: str | None = argv[2] except IndexError: env_script = None if env_script is not None: diff --git a/cwltool/secrets.py b/cwltool/secrets.py index 3d54774e1..6a39231e4 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -2,7 +2,6 @@ import uuid from collections.abc import MutableMapping, MutableSequence -from typing import Optional from .utils import CWLObjectType, CWLOutputType @@ -14,7 +13,7 @@ def __init__(self) -> None: """Initialize the secret store.""" self.secrets: dict[str, str] = {} - def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: + def add(self, value: CWLOutputType | None) -> CWLOutputType | None: """ Add the given value to the store. @@ -37,28 +36,30 @@ def store(self, secrets: list[str], job: CWLObjectType) -> None: def has_secret(self, value: CWLOutputType) -> bool: """Test if the provided document has any of our secrets.""" - if isinstance(value, str): - for k in self.secrets: - if k in value: - return True - elif isinstance(value, MutableMapping): - for this_value in value.values(): - if self.has_secret(this_value): - return True - elif isinstance(value, MutableSequence): - for this_value in value: - if self.has_secret(this_value): - return True + match value: + case str(val): + for k in self.secrets: + if k in val: + return True + case MutableMapping() as v_dict: + for this_value in v_dict.values(): + if self.has_secret(this_value): + return True + case MutableSequence() as seq: + for this_value in seq: + if self.has_secret(this_value): + return True return False def retrieve(self, value: CWLOutputType) -> CWLOutputType: """Replace placeholders with their corresponding secrets.""" - if isinstance(value, str): - for key, this_value in self.secrets.items(): - value = value.replace(key, this_value) - return value - elif isinstance(value, MutableMapping): - return {k: self.retrieve(v) for k, v in value.items()} - elif isinstance(value, MutableSequence): - return [self.retrieve(v) for v in value] + match value: + case str(val): + for key, this_value in self.secrets.items(): + val = val.replace(key, this_value) + return val + case MutableMapping() as v_dict: + return {k: self.retrieve(v) for k, v in v_dict.items()} + case MutableSequence() as seq: + return [self.retrieve(v) for v in seq] return value diff --git a/cwltool/singularity.py b/cwltool/singularity.py index d0e46fb27..76f1fc488 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,9 +6,9 @@ import re import shutil import sys -from collections.abc import MutableMapping +from collections.abc import Callable, MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Optional, cast +from typing import cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -29,7 +29,7 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[list[int]] = None +_SINGULARITY_VERSION: list[int] | None = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" @@ -348,7 +348,7 @@ def get_from_requirements( pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, - ) -> Optional[str]: + ) -> str | None: """ Return the filename of the Singularity image. @@ -383,7 +383,7 @@ def append_volume(runtime: list[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: str | None ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -401,7 +401,7 @@ def add_writable_file_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -438,7 +438,7 @@ def add_writable_directory_volume( self, runtime: list[str], volume: MapperEnt, - host_outdir_tgt: Optional[str], + host_outdir_tgt: str | None, tmpdir_prefix: str, ) -> None: if volume.resolved.startswith("_:"): @@ -479,7 +479,7 @@ def _required_env(self) -> dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> tuple[list[str], Optional[str]]: + ) -> tuple[list[str], str | None]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False @@ -499,7 +499,7 @@ def create_runtime( else: runtime.append("--pid") - container_HOME: Optional[str] = None + container_HOME: str | None = None if is_version_3_1_or_newer(): # Remove HOME, as passed in a special way (restore it below) container_HOME = self.environment.pop("HOME") diff --git a/cwltool/singularity_utils.py b/cwltool/singularity_utils.py index 13f7ed3f6..7f69eb2fc 100644 --- a/cwltool/singularity_utils.py +++ b/cwltool/singularity_utils.py @@ -3,9 +3,8 @@ import os import os.path import subprocess # nosec -from typing import Optional -_USERNS: Optional[bool] = None +_USERNS: bool | None = None def singularity_supports_userns() -> bool: diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index f66ad353c..307e98fd8 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -12,7 +12,7 @@ import os import string from collections.abc import MutableMapping, MutableSequence -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from .utils import HasReqsHints @@ -48,8 +48,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" - self.tool_dependency_dir: Optional[str] = None - self.dependency_resolvers_config_file: Optional[str] = None + self.tool_dependency_dir: str | None = None + self.dependency_resolvers_config_file: str | None = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) @@ -137,8 +137,8 @@ def get_dependencies( def get_container_from_software_requirements( - use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: Optional[str] = "." -) -> Optional[str]: + use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: str | None = "." +) -> str | None: if use_biocontainers: ensure_galaxy_lib_available() from galaxy.tool_util.deps.container_classes import DOCKER_CONTAINER_TYPE diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index dffa0cd85..761b83526 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -54,7 +54,7 @@ def listdir(self, fn: str) -> list[str]: """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] - def join(self, path, *paths): # type: (str, *str) -> str + def join(self, path: str, *paths: str) -> str: """Join one or more path segments intelligently.""" return os.path.join(path, *paths) diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 0a0289cc3..8bad72a47 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,12 +1,11 @@ import urllib from collections.abc import Mapping, MutableMapping, MutableSequence -from typing import Any, NamedTuple, Optional, Union, cast +from typing import Any, NamedTuple, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from .context import LoadingContext from .load_tool import load_tool, make_tool -from .process import Process from .utils import CWLObjectType, aslist from .workflow import Workflow, WorkflowStep @@ -14,7 +13,7 @@ class _Node(NamedTuple): up: list[str] down: list[str] - type: Optional[str] + type: str | None UP = "up" @@ -42,7 +41,7 @@ def _subgraph_visit( _subgraph_visit(c, nodes, visited, direction) -def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: Optional[str]) -> _Node: +def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: str | None) -> _Node: """ Record the given nodeid in the graph. @@ -60,50 +59,50 @@ def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: Optional[str]) -> _N def find_step( steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: +) -> tuple[CWLObjectType | None, WorkflowStep | None]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] if st_tool_id == stepid: return st.tool, st if stepid.startswith(st_tool_id): - run: Union[str, Process, CWLObjectType] = st.tool["run"] - if isinstance(run, Workflow): - result, st2 = find_step( - run.steps, stepid[len(st.tool["id"]) + 1 :], loading_context - ) - if result: - return result, st2 - elif isinstance(run, CommentedMap) and run["class"] == "Workflow": - process = make_tool(run, loading_context) - if isinstance(process, Workflow): - suffix = stepid[len(st.tool["id"]) + 1 :] - prefix = process.tool["id"] - if "#" in prefix: - sep = "/" - else: - sep = "#" - adj_stepid = f"{prefix}{sep}{suffix}" - result2, st3 = find_step( - process.steps, - adj_stepid, - loading_context, + match st.tool["run"]: + case Workflow(steps=steps): + result, st2 = find_step( + steps, stepid[len(st.tool["id"]) + 1 :], loading_context ) - if result2: - return result2, st3 - elif isinstance(run, str): - process = load_tool(run, loading_context) - if isinstance(process, Workflow): - suffix = stepid[len(st.tool["id"]) + 1 :] - prefix = process.tool["id"] - if "#" in prefix: - sep = "/" - else: - sep = "#" - adj_stepid = f"{prefix}{sep}{suffix}" - result3, st4 = find_step(process.steps, adj_stepid, loading_context) - if result3: - return result3, st4 + if result: + return result, st2 + case {"class": "Workflow"}: + process = make_tool(st.tool["run"], loading_context) + if isinstance(process, Workflow): + suffix = stepid[len(st.tool["id"]) + 1 :] + prefix = process.tool["id"] + if "#" in prefix: + sep = "/" + else: + sep = "#" + adj_stepid = f"{prefix}{sep}{suffix}" + result2, st3 = find_step( + process.steps, + adj_stepid, + loading_context, + ) + if result2: + return result2, st3 + case str(run_line): + process = load_tool(run_line, loading_context) + if isinstance(process, Workflow): + suffix = stepid[len(st.tool["id"]) + 1 :] + prefix = process.tool["id"] + if "#" in prefix: + sep = "/" + else: + sep = "#" + adj_stepid = f"{prefix}{sep}{suffix}" + result3, st4 = find_step(process.steps, adj_stepid, loading_context) + if result3: + return result3, st4 return None, None @@ -264,7 +263,7 @@ def get_process( if raw_step is None or step is None: raise Exception(f"Step {step_id} was not found") - run: Union[str, Any] = raw_step["run"] + run: str | Any = raw_step["run"] if isinstance(run, str): process = loading_context.loader.idx[run] diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index 59b1609e9..e86172a9a 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -5,7 +5,8 @@ import queue import threading -from typing import Callable, Optional +from collections.abc import Callable +from typing import Union from .loghandler import _logger @@ -36,12 +37,12 @@ class TaskQueue: def __init__(self, lock: threading.Lock, thread_count: int): """Create a new task queue using the specified lock and number of threads.""" self.thread_count = thread_count - self.task_queue: queue.Queue[Optional[Callable[[], None]]] = queue.Queue( + self.task_queue: queue.Queue[Callable[[], None] | None] = queue.Queue( maxsize=self.thread_count ) self.task_queue_threads = [] self.lock = lock - self.error: Optional[BaseException] = None + self.error: BaseException | None = None for _r in range(0, self.thread_count): t = threading.Thread(target=self._task_queue_func) @@ -65,8 +66,8 @@ def _task_queue_func(self) -> None: def add( self, task: Callable[[], None], - unlock: Optional[threading.Condition] = None, - check_done: Optional[threading.Event] = None, + unlock: Union[threading.Condition, None] = None, + check_done: Union[threading.Event, None] = None, ) -> None: """ Add your task to the queue. diff --git a/cwltool/update.py b/cwltool/update.py index 290498031..91a63f496 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,7 +1,7 @@ import copy -from collections.abc import MutableMapping, MutableSequence +from collections.abc import Callable, MutableMapping, MutableSequence from functools import partial -from typing import Callable, Optional, Union, cast +from typing import cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -74,7 +74,7 @@ def v1_1to1_2( """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) - upd: Union[CommentedSeq, CommentedMap] = doc + upd: CommentedSeq | CommentedMap = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -125,7 +125,7 @@ def rewrite_requirements(t: CWLObjectType) -> None: def update_secondaryFiles( t: CWLOutputType, top: bool = False - ) -> Union[MutableSequence[MutableMapping[str, str]], MutableMapping[str, str]]: + ) -> MutableSequence[MutableMapping[str, str]] | MutableMapping[str, str]: if isinstance(t, CommentedSeq): new_seq = copy.deepcopy(t) for index, entry in enumerate(t): @@ -158,7 +158,7 @@ def fix_inputBinding(t: CWLObjectType) -> None: visit_class(doc, ("ExpressionTool", "Workflow"), fix_inputBinding) visit_field(doc, "secondaryFiles", partial(update_secondaryFiles, top=True)) - upd: Union[CommentedMap, CommentedSeq] = doc + upd: CommentedMap | CommentedSeq = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -213,7 +213,7 @@ def update_pickvalue(t: CWLObjectType) -> None: inp["pickValue"] = "the_only_non_null" visit_class(doc, "Workflow", update_pickvalue) - upd: Union[CommentedSeq, CommentedMap] = doc + upd: CommentedSeq | CommentedMap = doc if isinstance(upd, MutableMapping) and "$graph" in upd: upd = upd["$graph"] for proc in aslist(upd): @@ -256,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -289,7 +289,7 @@ def identity( def checkversion( - doc: Union[CommentedSeq, CommentedMap], + doc: CommentedSeq | CommentedMap, metadata: CommentedMap, enable_dev: bool, ) -> tuple[CommentedMap, str]: @@ -297,7 +297,7 @@ def checkversion( Returns the document and the validated version string. """ - cdoc: Optional[CommentedMap] = None + cdoc: CommentedMap | None = None if isinstance(doc, CommentedSeq): if not isinstance(metadata, CommentedMap): raise Exception("Expected metadata to be CommentedMap") @@ -343,12 +343,12 @@ def checkversion( def update( - doc: Union[CommentedSeq, CommentedMap], + doc: CommentedSeq | CommentedMap, loader: Loader, baseuri: str, enable_dev: bool, metadata: CommentedMap, - update_to: Optional[str] = None, + update_to: str | None = None, ) -> CommentedMap: """Update a CWL document to 'update_to' (if provided) or INTERNAL_VERSION.""" if update_to is None: @@ -357,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]] = identity + nextupdate: Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]] | None = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c37dbe8e7..05913bf63 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,7 +19,13 @@ import tempfile import urllib import uuid -from collections.abc import Generator, Iterable, MutableMapping, MutableSequence +from collections.abc import ( + Callable, + Generator, + Iterable, + MutableMapping, + MutableSequence, +) from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial @@ -30,11 +36,11 @@ IO, TYPE_CHECKING, Any, - Callable, Deque, Literal, NamedTuple, Optional, + TypeAlias, TypedDict, Union, cast, @@ -45,15 +51,16 @@ from cachecontrol.caches import FileCache from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException -from schema_salad.ref_resolver import Loader if TYPE_CHECKING: + from schema_salad.ref_resolver import Loader + from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase from .stdfsaccess import StdFsAccess from .workflow_job import WorkflowJob -__random_outdir: Optional[str] = None +__random_outdir: str | None = None CONTENT_LIMIT = 64 * 1024 @@ -61,7 +68,7 @@ processes_to_kill: Deque["subprocess.Popen[str]"] = collections.deque() -CWLOutputType = Union[ +CWLOutputType: TypeAlias = Union[ None, bool, str, @@ -70,33 +77,35 @@ MutableSequence["CWLOutputType"], MutableMapping[str, "CWLOutputType"], ] -CWLObjectType = MutableMapping[str, Optional[CWLOutputType]] +CWLObjectType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] """Typical raw dictionary found in lightly parsed CWL.""" -JobsType = Union["CommandLineJob", "JobBase", "WorkflowJob", "ExpressionJob", "CallbackJob"] -JobsGeneratorType = Generator[Optional[JobsType], None, None] -OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] -ResolverType = Callable[["Loader", str], Optional[str]] -DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, list[Optional[CWLOutputType]]] -ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] -SinkType = Union[CWLOutputType, CWLObjectType] +JobsType: TypeAlias = Union[ + "CommandLineJob", "JobBase", "WorkflowJob", "ExpressionJob", "CallbackJob" +] +JobsGeneratorType: TypeAlias = Generator[Optional[JobsType], None, None] +OutputCallbackType: TypeAlias = Callable[[Optional[CWLObjectType], str], None] +ResolverType: TypeAlias = Callable[["Loader", str], Optional[str]] +DestinationsType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] +ScatterDestinationsType: TypeAlias = MutableMapping[str, list[Optional[CWLOutputType]]] +ScatterOutputCallbackType: TypeAlias = Callable[[Optional[ScatterDestinationsType], str], None] +SinkType: TypeAlias = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] +JSONType: TypeAlias = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): """Workflow state item.""" parameter: CWLObjectType - value: Optional[CWLOutputType] + value: CWLOutputType | None success: str -ParametersType = list[CWLObjectType] -StepType = CWLObjectType # WorkflowStep +ParametersType: TypeAlias = list[CWLObjectType] +StepType: TypeAlias = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -164,8 +173,8 @@ def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: def bytes2str_in_dicts( - inp: Union[MutableMapping[str, Any], MutableSequence[Any], Any], -) -> Union[str, MutableSequence[Any], MutableMapping[str, Any]]: + inp: MutableMapping[str, Any] | MutableSequence[Any] | Any, +) -> str | MutableSequence[Any] | MutableMapping[str, Any]: """ Convert any present byte string to unicode string, inplace. @@ -308,7 +317,7 @@ def trim_listing(obj: dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, datetime | None]: """ Download a remote file, possibly using a locally cached copy. @@ -336,8 +345,8 @@ def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: f.write(chunk) r.close() - date_raw: Optional[str] = r.headers.get("Last-Modified", None) - date: Optional[datetime] = parsedate_to_datetime(date_raw) if date_raw else None + date_raw: str | None = r.headers.get("Last-Modified", None) + date: datetime | None = parsedate_to_datetime(date_raw) if date_raw else None if date: date_epoch = date.timestamp() os.utime(f.name, (date_epoch, date_epoch)) @@ -392,13 +401,9 @@ def ensure_non_writable(path: str) -> None: def normalizeFilesDirs( - job: Optional[ - Union[ - MutableSequence[MutableMapping[str, Any]], - MutableMapping[str, Any], - DirectoryType, - ] - ], + job: None | ( + MutableSequence[MutableMapping[str, Any]] | MutableMapping[str, Any] | DirectoryType + ), ) -> None: def addLocation(d: dict[str, Any]) -> None: if "location" not in d: @@ -473,7 +478,7 @@ def __init__(self) -> None: self.requirements: list[CWLObjectType] = [] self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[CWLObjectType | None, bool | None]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 3a490b68d..4aafc497b 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -4,7 +4,7 @@ import logging from collections.abc import MutableMapping, MutableSequence from importlib.resources import files -from typing import Any, NamedTuple, Optional, Union, cast +from typing import Any, NamedTuple, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -25,7 +25,7 @@ from .loghandler import _logger -def is_expression(tool: Any, schema: Optional[Schema]) -> bool: +def is_expression(tool: Any, schema: Schema | None) -> bool: """Test a field/schema combo to see if it is a CWL Expression.""" return ( isinstance(schema, EnumSchema) @@ -50,63 +50,62 @@ def filter(self, record: logging.LogRecord) -> bool: def get_expressions( - tool: Union[CommentedMap, str, CommentedSeq], - schema: Optional[Union[Schema, ArraySchema]], - source_line: Optional[SourceLine] = None, -) -> list[tuple[str, Optional[SourceLine]]]: + tool: CommentedMap | str | CommentedSeq, + schema: Schema | ArraySchema | None, + source_line: SourceLine | None = None, +) -> list[tuple[str, SourceLine | None]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] - elif isinstance(schema, UnionSchema): - valid_schema = None - - for possible_schema in schema.schemas: - if is_expression(tool, possible_schema): - return [(cast(str, tool), source_line)] - elif validate_ex( - possible_schema, - tool, - raise_ex=False, - logger=_logger_validation_warnings, - vocab={}, - ): - valid_schema = possible_schema - - return get_expressions(tool, valid_schema, source_line) - elif isinstance(schema, ArraySchema): - if not isinstance(tool, MutableSequence): - return [] - - return list( - itertools.chain( - *map( - lambda x: get_expressions( - x[1], getattr(schema, "items"), SourceLine(tool, x[0]) # noqa: B009 - ), - enumerate(tool), + match schema: + case UnionSchema(schemas=schemas): + valid_schema = None + + for possible_schema in schemas: + if is_expression(tool, possible_schema): + return [(cast(str, tool), source_line)] + elif validate_ex( + possible_schema, + tool, + raise_ex=False, + logger=_logger_validation_warnings, + vocab={}, + ): + valid_schema = possible_schema + + return get_expressions(tool, valid_schema, source_line) + case ArraySchema(items=items): + if not isinstance(tool, MutableSequence): + return [] + + return list( + itertools.chain( + *map( + lambda x: get_expressions( + x[1], items, SourceLine(tool, x[0]) # noqa: B009 + ), + enumerate(tool), + ) ) ) - ) - - elif isinstance(schema, RecordSchema): - if not isinstance(tool, MutableMapping): - return [] - - expression_nodes = [] - - for schema_field in schema.fields: - if schema_field.name in tool: - expression_nodes.extend( - get_expressions( - tool[schema_field.name], - schema_field.type, - SourceLine(tool, schema_field.name, include_traceback=debug), + case RecordSchema(fields=fields): + if not isinstance(tool, MutableMapping): + return [] + + expression_nodes = [] + + for schema_field in fields: + if schema_field.name in tool: + expression_nodes.extend( + get_expressions( + tool[schema_field.name], + schema_field.type, + SourceLine(tool, schema_field.name, include_traceback=debug), + ) ) - ) - - return expression_nodes - else: - return [] + return expression_nodes + case _: + return [] class JSHintJSReturn(NamedTuple): @@ -118,8 +117,8 @@ class JSHintJSReturn(NamedTuple): def jshint_js( js_text: str, - globals: Optional[list[str]] = None, - options: Optional[dict[str, Union[list[str], str, int]]] = None, + globals: list[str] | None = None, + options: dict[str, list[str] | str | int] | None = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -187,7 +186,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: SourceLine | None) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -197,7 +196,7 @@ def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[dict[str, Union[list[str], str, int]]] = None, + jshint_options: dict[str, list[str] | str | int] | None = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index a6e2ba189..fe34c75ce 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,8 +3,8 @@ import functools import logging import random -from collections.abc import Mapping, MutableMapping, MutableSequence -from typing import Callable, Optional, cast +from collections.abc import Callable, Mapping, MutableMapping, MutableSequence +from typing import cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -33,24 +33,25 @@ def default_make_tool(toolpath_object: CommentedMap, loadingContext: LoadingContext) -> Process: """Instantiate the given CWL Process.""" - if not isinstance(toolpath_object, MutableMapping): - raise WorkflowException("Not a dict: '%s'" % toolpath_object) - if "class" in toolpath_object: - if toolpath_object["class"] == "CommandLineTool": + match toolpath_object: + case {"class": "CommandLineTool"}: return command_line_tool.CommandLineTool(toolpath_object, loadingContext) - if toolpath_object["class"] == "ExpressionTool": + case {"class": "ExpressionTool"}: return command_line_tool.ExpressionTool(toolpath_object, loadingContext) - if toolpath_object["class"] == "Workflow": + case {"class": "Workflow"}: return Workflow(toolpath_object, loadingContext) - if toolpath_object["class"] == "ProcessGenerator": + case {"class": "ProcessGenerator"}: return procgenerator.ProcessGenerator(toolpath_object, loadingContext) - if toolpath_object["class"] == "Operation": + case {"class": "Operation"}: return command_line_tool.AbstractOperation(toolpath_object, loadingContext) - - raise WorkflowException( - "Missing or invalid 'class' field in " - "%s, expecting one of: CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"] - ) + case MutableMapping(): + raise WorkflowException( + "Missing or invalid 'class' field in " + f"{toolpath_object['id']}, expecting one of: CommandLineTool, " + "ExpressionTool, Workflow" + ) + case _: + raise WorkflowException("Not a dict: '%s'" % toolpath_object) context.default_make_tool = default_make_tool @@ -65,9 +66,9 @@ def __init__( ) -> None: """Initialize this Workflow.""" super().__init__(toolpath_object, loadingContext) - self.provenance_object: Optional[ProvenanceProfile] = None + self.provenance_object: ProvenanceProfile | None = None if loadingContext.research_obj is not None: - run_uuid: Optional[UUID] = None + run_uuid: UUID | None = None is_main = not loadingContext.prov_obj # Not yet set if is_main: run_uuid = loadingContext.research_obj.ro_uuid @@ -137,7 +138,7 @@ def make_workflow_step( toolpath_object: CommentedMap, pos: int, loadingContext: LoadingContext, - parentworkflowProv: Optional[ProvenanceProfile] = None, + parentworkflowProv: ProvenanceProfile | None = None, ) -> "WorkflowStep": return WorkflowStep(toolpath_object, pos, loadingContext, parentworkflowProv) @@ -187,7 +188,7 @@ def __init__( toolpath_object: CommentedMap, pos: int, loadingContext: LoadingContext, - parentworkflowProv: Optional[ProvenanceProfile] = None, + parentworkflowProv: ProvenanceProfile | None = None, ) -> None: """Initialize this WorkflowStep.""" debug = loadingContext.debug @@ -385,7 +386,7 @@ def __init__( oparam["type"] = {"type": "array", "items": oparam["type"]} self.tool["inputs"] = inputparms self.tool["outputs"] = outputparms - self.prov_obj: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None if loadingContext.research_obj is not None: self.prov_obj = parentworkflowProv if self.embedded_tool.tool["class"] == "Workflow": diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 368d1cc02..a85dc81f8 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -46,7 +46,7 @@ def __init__(self, step: "WorkflowStep") -> None: self.tool = step.tool self.id = step.id self.submitted = False - self.iterable: Optional[JobsGeneratorType] = None + self.iterable: JobsGeneratorType | None = None self.completed = False self.name = uniquename("step %s" % shortname(self.id)) self.prov_obj = step.prov_obj @@ -87,7 +87,7 @@ def __init__( self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: list[Optional[JobsGeneratorType]] = [] + self.steps: list[JobsGeneratorType | None] = [] @property def completed(self) -> int: @@ -117,7 +117,7 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu def setTotal( self, total: int, - steps: list[Optional[JobsGeneratorType]], + steps: list[JobsGeneratorType | None], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -131,7 +131,7 @@ def setTotal( def parallel_steps( - steps: list[Optional[JobsGeneratorType]], + steps: list[JobsGeneratorType | None], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -181,9 +181,9 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: list[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] for index in range(0, jobl): - sjob: Optional[CWLObjectType] = copy.copy(joborder) + sjob: CWLObjectType | None = copy.copy(joborder) assert sjob is not None # nosec sjob[scatter_key] = cast(MutableMapping[int, CWLObjectType], joborder[scatter_key])[index] @@ -248,14 +248,14 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> tuple[list[Optional[JobsGeneratorType]], int]: +) -> tuple[list[JobsGeneratorType | None], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: list[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] put = startindex for index in range(0, jobl): - sjob: Optional[CWLObjectType] = copy.copy(joborder) + sjob: CWLObjectType | None = copy.copy(joborder) assert sjob is not None # nosec sjob[scatter_key] = cast(MutableMapping[int, CWLObjectType], joborder[scatter_key])[index] @@ -286,7 +286,7 @@ def dotproduct_scatter( output_callback: ScatterOutputCallbackType, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: - jobl: Optional[int] = None + jobl: int | None = None for key in scatter_keys: if jobl is None: jobl = len(cast(Sized, joborder[key])) @@ -303,9 +303,9 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: list[Optional[JobsGeneratorType]] = [] + steps: list[JobsGeneratorType | None] = [] for index in range(0, jobl): - sjobo: Optional[CWLObjectType] = copy.copy(joborder) + sjobo: CWLObjectType | None = copy.copy(joborder) assert sjobo is not None # nosec for key in scatter_keys: sjobo[key] = cast(MutableMapping[int, CWLObjectType], joborder[key])[index] @@ -324,12 +324,12 @@ def dotproduct_scatter( def match_types( - sinktype: Optional[SinkType], + sinktype: SinkType | None, src: WorkflowStateItem, iid: str, inputobj: CWLObjectType, - linkMerge: Optional[str], - valueFrom: Optional[str], + linkMerge: str | None, + valueFrom: str | None, ) -> bool: if isinstance(sinktype, MutableSequence): # Sink is union type @@ -374,13 +374,13 @@ def match_types( def object_from_state( - state: dict[str, Optional[WorkflowStateItem]], + state: dict[str, WorkflowStateItem | None], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, sourceField: str, incomplete: bool = False, -) -> Optional[CWLObjectType]: +) -> CWLObjectType | None: inputobj: CWLObjectType = {} for inp in params: iid = original_id = cast(str, inp["id"]) @@ -474,17 +474,17 @@ class WorkflowJob: def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None: """Initialize this WorkflowJob.""" self.workflow = workflow - self.prov_obj: Optional[ProvenanceProfile] = None - self.parent_wf: Optional[ProvenanceProfile] = None + self.prov_obj: ProvenanceProfile | None = None + self.parent_wf: ProvenanceProfile | None = None self.tool = workflow.tool if runtimeContext.research_obj is not None: self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, WorkflowStateItem | None] = {} self.processStatus = "" self.did_callback = False - self.made_progress: Optional[bool] = None + self.made_progress: bool | None = None self.outdir = runtimeContext.get_outdir() self.name = uniquename( @@ -507,7 +507,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: self.workflow.get_requirement("MultipleInputFeatureRequirement")[0] ) - wo: Optional[CWLObjectType] = None + wo: CWLObjectType | None = None try: wo = object_from_state( self.state, @@ -525,7 +525,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: and self.parent_wf and self.prov_obj.workflow_run_uri != self.parent_wf.workflow_run_uri ): - process_run_id: Optional[str] = None + process_run_id: str | None = None self.prov_obj.generate_output_prov(wo or {}, process_run_id, self.name) self.prov_obj.document.wasEndedBy( self.prov_obj.workflow_run_uri, @@ -628,7 +628,7 @@ def try_make_job( "Workflow step contains valueFrom but StepInputExpressionRequirement not in requirements" ) - def postScatterEval(io: CWLObjectType) -> Optional[CWLObjectType]: + def postScatterEval(io: CWLObjectType) -> CWLObjectType | None: shortio = cast(CWLObjectType, {shortname(k): v for k, v in io.items()}) fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess)("") @@ -639,7 +639,7 @@ def postScatterEval(io: CWLObjectType) -> Optional[CWLObjectType]: with fs_access.open(cast(str, val["location"]), "rb") as f: val["contents"] = content_limit_respected_read(f) - def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType]: + def valueFromFunc(k: str, v: CWLOutputType | None) -> CWLOutputType | None: if k in valueFrom: adjustDirObjs(v, functools.partial(get_listing, fs_access, recursive=True)) @@ -712,17 +712,17 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] step.name, "', '".join(emptyscatter), ) - - if method == "dotproduct" or method is None: - jobs = dotproduct_scatter(step, inputobj, scatter, callback, runtimeContext) - elif method == "nested_crossproduct": - jobs = nested_crossproduct_scatter( - step, inputobj, scatter, callback, runtimeContext - ) - elif method == "flat_crossproduct": - jobs = flat_crossproduct_scatter( - step, inputobj, scatter, callback, runtimeContext - ) + match method: + case "dotproduct" | None: + jobs = dotproduct_scatter(step, inputobj, scatter, callback, runtimeContext) + case "nested_crossproduct": + jobs = nested_crossproduct_scatter( + step, inputobj, scatter, callback, runtimeContext + ) + case "flat_crossproduct": + jobs = flat_crossproduct_scatter( + step, inputobj, scatter, callback, runtimeContext + ) else: if _logger.isEnabledFor(logging.DEBUG): _logger.debug("[%s] job input %s", step.name, json_dumps(inputobj, indent=4)) @@ -766,7 +766,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Optional[threading.Lock] = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: """Log the start of each workflow.""" _logger.info("[%s] start", self.name) @@ -863,12 +863,12 @@ def __init__(self, step: WorkflowJobStep, container_engine: str): """Initialize this WorkflowJobLoopStep.""" self.step: WorkflowJobStep = step self.container_engine: str = container_engine - self.joborder: Optional[CWLObjectType] = None + self.joborder: CWLObjectType | None = None self.processStatus: str = "success" self.iteration: int = 0 self.output_buffer: MutableMapping[ str, - Union[MutableSequence[Optional[CWLOutputType]], Optional[CWLOutputType]], + MutableSequence[CWLOutputType | None] | CWLOutputType | None, ] = {} def _set_empty_output(self, outputMethod: str) -> None: @@ -955,7 +955,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, WorkflowStateItem | None] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/mypy-stubs/arcp/parse.pyi b/mypy-stubs/arcp/parse.pyi index 560671663..3ff469307 100644 --- a/mypy-stubs/arcp/parse.pyi +++ b/mypy-stubs/arcp/parse.pyi @@ -22,4 +22,4 @@ class ARCPParseResult(ParseResult): def nih_uri(self) -> str | None: ... def ni_well_known(self, base: str = ...) -> str | None: ... @property - def hash(self) -> Tuple[str, str] | None: ... + def hash(self) -> tuple[str, str] | None: ... diff --git a/mypy-stubs/bagit.pyi b/mypy-stubs/bagit.pyi index 2b0f5b9f2..ff9a7cfbc 100644 --- a/mypy-stubs/bagit.pyi +++ b/mypy-stubs/bagit.pyi @@ -1,5 +1,6 @@ import argparse -from typing import Any, Dict, Iterator, List, Optional, Tuple +from collections.abc import Iterator +from typing import Any, Dict, List, Optional, Tuple from _typeshed import Incomplete @@ -19,28 +20,28 @@ class Bag: valid_files: Any = ... valid_directories: Any = ... tags: Any = ... - info: Dict[str, str] = ... + info: dict[str, str] = ... entries: Any = ... normalized_filesystem_names: Any = ... normalized_manifest_names: Any = ... algorithms: Any = ... tag_file_name: Any = ... path: Any = ... - def __init__(self, path: Optional[Any] = ...) -> None: ... + def __init__(self, path: Any | None = ...) -> None: ... @property - def algs(self) -> List[str]: ... + def algs(self) -> list[str]: ... @property def version(self) -> str: ... def manifest_files(self) -> Iterator[str]: ... def tagmanifest_files(self) -> None: ... - def compare_manifests_with_fs(self) -> Tuple[List[str], List[str]]: ... - def compare_fetch_with_fs(self) -> List[str]: ... + def compare_manifests_with_fs(self) -> tuple[list[str], list[str]]: ... + def compare_fetch_with_fs(self) -> list[str]: ... def payload_files(self) -> Iterator[str]: ... - def payload_entries(self) -> Dict[str, str]: ... + def payload_entries(self) -> dict[str, str]: ... def save(self, processes: int = ..., manifests: bool = ...) -> None: ... - def tagfile_entries(self) -> Dict[str, str]: ... + def tagfile_entries(self) -> dict[str, str]: ... def missing_optional_tagfiles(self) -> Iterator[str]: ... - def fetch_entries(self) -> Iterator[Tuple[str, str, str]]: ... + def fetch_entries(self) -> Iterator[tuple[str, str, str]]: ... def files_to_be_fetched(self) -> Iterator[str]: ... def has_oxum(self) -> bool: ... def validate( diff --git a/mypy-stubs/prov/constants.py b/mypy-stubs/prov/constants.py index 314224a63..67980a67e 100644 --- a/mypy-stubs/prov/constants.py +++ b/mypy-stubs/prov/constants.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - from prov.identifier import Namespace __author__ = "Trung Dong Huynh" @@ -170,13 +168,9 @@ PROV_ATTRIBUTES = PROV_ATTRIBUTE_QNAMES | PROV_ATTRIBUTE_LITERALS PROV_RECORD_ATTRIBUTES = list((attr, str(attr)) for attr in PROV_ATTRIBUTES) -PROV_RECORD_IDS_MAP = dict((PROV_N_MAP[rec_type_id], rec_type_id) for rec_type_id in PROV_N_MAP) -PROV_ID_ATTRIBUTES_MAP = dict( - (prov_id, attribute) for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES -) -PROV_ATTRIBUTES_ID_MAP = dict( - (attribute, prov_id) for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES -) +PROV_RECORD_IDS_MAP = {PROV_N_MAP[rec_type_id]: rec_type_id for rec_type_id in PROV_N_MAP} +PROV_ID_ATTRIBUTES_MAP = {prov_id: attribute for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES} +PROV_ATTRIBUTES_ID_MAP = {attribute: prov_id for (prov_id, attribute) in PROV_RECORD_ATTRIBUTES} # Extra definition for convenience PROV_TYPE = PROV["type"] diff --git a/mypy-stubs/prov/identifier.pyi b/mypy-stubs/prov/identifier.pyi index c0d79d912..c344128cd 100644 --- a/mypy-stubs/prov/identifier.pyi +++ b/mypy-stubs/prov/identifier.pyi @@ -23,8 +23,8 @@ class Namespace: def uri(self) -> str: ... @property def prefix(self) -> str: ... - def contains(self, identifier: Union[Identifier, str]) -> bool: ... - def qname(self, identifier: Union[Identifier, str]) -> QualifiedName: ... + def contains(self, identifier: Identifier | str) -> bool: ... + def qname(self, identifier: Identifier | str) -> QualifiedName: ... def __eq__(self, other: Any) -> bool: ... def __ne__(self, other: Any) -> bool: ... def __hash__(self) -> int: ... diff --git a/mypy-stubs/prov/model.pyi b/mypy-stubs/prov/model.pyi index 19a13bcf0..e1147872a 100644 --- a/mypy-stubs/prov/model.pyi +++ b/mypy-stubs/prov/model.pyi @@ -1,5 +1,6 @@ +from collections.abc import Iterable from datetime import datetime -from typing import IO, Any, Dict, Iterable, List, Set, Tuple +from typing import IO, Any, Dict, List, Set, Tuple from _typeshed import Incomplete from prov.constants import * @@ -16,7 +17,7 @@ XSD_DATATYPE_PARSERS: Incomplete # def first(a_set): ... -_attributes_type = Dict[str | Identifier, Any] | List[Tuple[str | Identifier, Any]] +_attributes_type = dict[str | Identifier, Any] | list[tuple[str | Identifier, Any]] class ProvRecord: FORMAL_ATTRIBUTES: Incomplete @@ -24,24 +25,24 @@ class ProvRecord: self, bundle: str, identifier: Identifier | str, - attributes: Dict[str, str] | None = ..., + attributes: dict[str, str] | None = ..., ) -> None: ... def __hash__(self) -> int: ... def copy(self) -> ProvRecord: ... def get_type(self) -> str: ... - def get_asserted_types(self) -> Set[str]: ... + def get_asserted_types(self) -> set[str]: ... def add_asserted_type(self, type_identifier: str | QualifiedName) -> None: ... - def get_attribute(self, attr_name: str) -> Set[str]: ... + def get_attribute(self, attr_name: str) -> set[str]: ... @property def identifier(self) -> Identifier: ... @property - def attributes(self) -> List[Tuple[str, str]]: ... + def attributes(self) -> list[tuple[str, str]]: ... @property - def args(self) -> Tuple[str, ...]: ... + def args(self) -> tuple[str, ...]: ... @property - def formal_attributes(self) -> Tuple[Tuple[str, str], ...]: ... + def formal_attributes(self) -> tuple[tuple[str, str], ...]: ... @property - def extra_attributes(self) -> Tuple[Tuple[str, str], ...]: ... + def extra_attributes(self) -> tuple[tuple[str, str], ...]: ... @property def bundle(self) -> "ProvBundle": ... @property @@ -203,7 +204,7 @@ class ProvBundle: document: ProvDocument | None = ..., ) -> None: ... @property - def namespaces(self) -> Set[Namespace]: ... + def namespaces(self) -> set[Namespace]: ... @property def default_ns_uri(self) -> str | None: ... @property @@ -211,7 +212,7 @@ class ProvBundle: @property def identifier(self) -> str | None | QualifiedName: ... @property - def records(self) -> List[ProvRecord]: ... + def records(self) -> list[ProvRecord]: ... def set_default_namespace(self, uri: Namespace) -> None: ... def get_default_namespace(self) -> Namespace: ... def add_namespace( @@ -219,15 +220,15 @@ class ProvBundle: ) -> Namespace: ... def get_registered_namespaces(self) -> Iterable[Namespace]: ... def valid_qualified_name( - self, identifier: QualifiedName | Tuple[str, Identifier] + self, identifier: QualifiedName | tuple[str, Identifier] ) -> QualifiedName | None: ... def get_records( self, class_or_type_or_tuple: ( - type | type[int | str] | Tuple[type | type[int | str] | Tuple[Any, ...], ...] | None + type | type[int | str] | tuple[type | type[int | str] | tuple[Any, ...], ...] | None ) = ..., - ) -> List[ProvRecord]: ... - def get_record(self, identifier: Identifier | None) -> ProvRecord | List[ProvRecord] | None: ... + ) -> list[ProvRecord]: ... + def get_record(self, identifier: Identifier | None) -> ProvRecord | list[ProvRecord] | None: ... def is_document(self) -> bool: ... def is_bundle(self) -> bool: ... def has_bundles(self) -> bool: ... @@ -452,4 +453,4 @@ class ProvDocument(ProvBundle): **args: Any, ) -> ProvDocument: ... -def sorted_attributes(element: ProvElement, attributes: List[str]) -> List[str]: ... +def sorted_attributes(element: ProvElement, attributes: list[str]) -> list[str]: ... diff --git a/mypy-stubs/rdflib/collection.pyi b/mypy-stubs/rdflib/collection.pyi index 0bee98429..a04649b6a 100644 --- a/mypy-stubs/rdflib/collection.pyi +++ b/mypy-stubs/rdflib/collection.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterator +from collections.abc import Iterator +from typing import Any from rdflib.graph import Graph from rdflib.term import Node diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index 9764972b2..93f3e6eae 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -1,22 +1,13 @@ import pathlib -from typing import ( - IO, - Any, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Union, - overload, -) +from builtins import set as _set +from collections.abc import Iterable, Iterator +from typing import IO, Any, List, Optional, Tuple, Union, overload from rdflib import query from rdflib.collection import Collection from rdflib.paths import Path from rdflib.resource import Resource -from rdflib.term import BNode, Identifier, Literal, Node +from rdflib.term import BNode, Identifier, Node class Graph(Node): base: Any = ... @@ -26,9 +17,9 @@ class Graph(Node): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - namespace_manager: Optional[Any] = ..., - base: Optional[Any] = ..., + identifier: Any | None = ..., + namespace_manager: Any | None = ..., + base: Any | None = ..., ) -> None: ... store: Any = ... identifier: Any = ... @@ -44,59 +35,53 @@ class Graph(Node): def remove(self, triple: Any) -> None: ... def triples( self, - triple: Tuple[ - Optional[Union[str, Identifier]], - Optional[Union[str, Identifier]], - Optional[Identifier], + triple: tuple[ + str | Identifier | None, + str | Identifier | None, + Identifier | None, ], - ) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... + ) -> Iterator[tuple[Identifier, Identifier, Identifier]]: ... def __getitem__( self, item: slice | Path | Node ) -> Iterator[ - Tuple[Identifier, Identifier, Identifier] | Tuple[Identifier, identifier] | Node + tuple[Identifier, Identifier, Identifier] | tuple[Identifier, identifier] | Node ]: ... def __contains__(self, triple: Any) -> bool: ... def __add__(self, other: Any) -> Graph: ... def set(self, triple: Any) -> None: ... - def subjects( - self, predicate: Optional[Any] = ..., object: Optional[Any] = ... - ) -> Iterable[Node]: ... - def predicates( - self, subject: Optional[Any] = ..., object: Optional[Any] = ... - ) -> Iterable[Node]: ... + def subjects(self, predicate: Any | None = ..., object: Any | None = ...) -> Iterable[Node]: ... + def predicates(self, subject: Any | None = ..., object: Any | None = ...) -> Iterable[Node]: ... def objects( - self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... - ) -> Iterable[Union[Identifier, Literal]]: ... - def subject_predicates(self, object: Optional[Any] = ...) -> None: ... - def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... - def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... + self, subject: Any | None = ..., predicate: Any | None = ... + ) -> Iterable[Identifier]: ... + def subject_predicates(self, object: Any | None = ...) -> None: ... + def subject_objects(self, predicate: Any | None = ...) -> None: ... + def predicate_objects(self, subject: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... def value( self, - subject: Optional[Any] = ..., + subject: Any | None = ..., predicate: Any = ..., - object: Optional[Any] = ..., - default: Optional[Any] = ..., + object: Any | None = ..., + default: Any | None = ..., any: bool = ..., ) -> Any: ... def label(self, subject: Any, default: str = ...) -> Any: ... def preferredLabel( self, subject: Any, - lang: Optional[Any] = ..., - default: Optional[Any] = ..., + lang: Any | None = ..., + default: Any | None = ..., labelProperties: Any = ..., - ) -> List[Tuple[Any, Any]]: ... + ) -> list[tuple[Any, Any]]: ... def comment(self, subject: Any, default: str = ...) -> Any: ... def items(self, list: Any) -> Iterator[Any]: ... - def transitiveClosure( - self, func: Any, arg: Any, seen: Optional[Any] = ... - ) -> Iterator[Any]: ... + def transitiveClosure(self, func: Any, arg: Any, seen: Any | None = ...) -> Iterator[Any]: ... def transitive_objects( - self, subject: Any, property: Any, remember: Optional[Any] = ... + self, subject: Any, property: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def transitive_subjects( - self, predicate: Any, object: Any, remember: Optional[Any] = ... + self, predicate: Any, object: Any, remember: Any | None = ... ) -> Iterator[Any]: ... def seq(self, subject: Any) -> Seq | None: ... def qname(self, uri: Any) -> Any: ... @@ -104,7 +89,7 @@ class Graph(Node): def bind( self, prefix: Any, namespace: Any, override: bool = ..., replace: bool = ... ) -> Any: ... - def namespaces(self) -> Iterator[Tuple[Any, Any]]: ... + def namespaces(self) -> Iterator[tuple[Any, Any]]: ... def absolutize(self, uri: Any, defrag: int = ...) -> Any: ... # no destination and non-None positional encoding @@ -113,7 +98,7 @@ class Graph(Node): self, destination: None, format: str, - base: Optional[str], + base: str | None, encoding: str, **args: Any, ) -> bytes: ... @@ -124,7 +109,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., *, encoding: str, **args: Any, @@ -136,7 +121,7 @@ class Graph(Node): self, destination: None = ..., format: str = ..., - base: Optional[str] = ..., + base: str | None = ..., encoding: None = ..., **args: Any, ) -> str: ... @@ -145,10 +130,10 @@ class Graph(Node): @overload def serialize( self, - destination: Union[str, pathlib.PurePath, IO[bytes]], + destination: str | pathlib.PurePath | IO[bytes], format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> "Graph": ... @@ -156,30 +141,30 @@ class Graph(Node): @overload def serialize( self, - destination: Optional[Union[str, pathlib.PurePath, IO[bytes]]] = ..., + destination: str | pathlib.PurePath | IO[bytes] | None = ..., format: str = ..., - base: Optional[str] = ..., - encoding: Optional[str] = ..., + base: str | None = ..., + encoding: str | None = ..., **args: Any, ) -> Union[bytes, str, "Graph"]: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> "Graph": ... - def load(self, source: Any, publicID: Optional[Any] = ..., format: str = ...) -> "Graph": ... + def load(self, source: Any, publicID: Any | None = ..., format: str = ...) -> "Graph": ... def query( self, query_object: Any, processor: str = ..., result: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> query.Result: ... @@ -187,27 +172,25 @@ class Graph(Node): self, update_object: Any, processor: str = ..., - initNs: Optional[Any] = ..., - initBindings: Optional[Any] = ..., + initNs: Any | None = ..., + initBindings: Any | None = ..., use_store_provided: bool = ..., **kwargs: Any, ) -> Any: ... def n3(self) -> str: ... def isomorphic(self, other: Any) -> bool: ... def connected(self) -> bool: ... - def all_nodes(self) -> Set[Any]: ... + def all_nodes(self) -> _set[Node]: ... def collection(self, identifier: Any) -> Collection: ... def resource(self, identifier: Any) -> Resource: ... def skolemize( self, - new_graph: Optional[Any] = ..., - bnode: Optional[Any] = ..., - authority: Optional[Any] = ..., - basepath: Optional[Any] = ..., - ) -> Graph: ... - def de_skolemize( - self, new_graph: Optional[Any] = ..., uriref: Optional[Any] = ... + new_graph: Any | None = ..., + bnode: Any | None = ..., + authority: Any | None = ..., + basepath: Any | None = ..., ) -> Graph: ... + def de_skolemize(self, new_graph: Any | None = ..., uriref: Any | None = ...) -> Graph: ... class ConjunctiveGraph(Graph): context_aware: bool = ... @@ -216,32 +199,32 @@ class ConjunctiveGraph(Graph): def __init__( self, store: str = ..., - identifier: Optional[Any] = ..., - default_graph_base: Optional[Any] = ..., + identifier: Any | None = ..., + default_graph_base: Any | None = ..., ) -> None: ... def add(self, triple_or_quad: Any) -> None: ... def addN(self, quads: Any) -> None: ... def remove(self, triple_or_quad: Any) -> None: ... # def triples(self, triple_or_quad: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]], context: Tuple[Optional[Union[str, BNode]], Optional[Union[str, BNode]], Optional[BNode]]) -> Iterator[Tuple[Identifier, Identifier, Identifier]]: ... - def quads(self, triple_or_quad: Optional[Any] = ...) -> None: ... - def triples_choices(self, triple: Any, context: Optional[Any] = ...) -> None: ... - def contexts(self, triple: Optional[Any] = ...) -> None: ... + def quads(self, triple_or_quad: Any | None = ...) -> None: ... + def triples_choices(self, triple: Any, context: Any | None = ...) -> None: ... + def contexts(self, triple: Any | None = ...) -> None: ... def get_context( self, identifier: Node | str | None, quoted: bool = ..., - base: Optional[str] = ..., + base: str | None = ..., ) -> Graph: ... def remove_context(self, context: Any) -> None: ... - def context_id(self, uri: Any, context_id: Optional[Any] = ...) -> Any: ... + def context_id(self, uri: Any, context_id: Any | None = ...) -> Any: ... def parse( self, - source: Optional[Any] = ..., - publicID: Optional[Any] = ..., - format: Optional[str] = ..., - location: Optional[Any] = ..., - file: Optional[Any] = ..., - data: Optional[Any] = ..., + source: Any | None = ..., + publicID: Any | None = ..., + format: str | None = ..., + location: Any | None = ..., + file: Any | None = ..., + data: Any | None = ..., **args: Any, ) -> Graph: ... diff --git a/mypy-stubs/rdflib/namespace/__init__.pyi b/mypy-stubs/rdflib/namespace/__init__.pyi index 63af92977..6f1b96f9d 100644 --- a/mypy-stubs/rdflib/namespace/__init__.pyi +++ b/mypy-stubs/rdflib/namespace/__init__.pyi @@ -59,7 +59,7 @@ SPLIT_START_CATEGORIES = NAME_START_CATEGORIES + ["Nd"] XMLNS = "http://www.w3.org/XML/1998/namespace" -def split_uri(uri: Any, split_start: Any = ...) -> Tuple[str, str]: ... +def split_uri(uri: Any, split_start: Any = ...) -> tuple[str, str]: ... from rdflib.namespace._CSVW import CSVW from rdflib.namespace._DC import DC diff --git a/mypy-stubs/rdflib/paths.pyi b/mypy-stubs/rdflib/paths.pyi index 9bea17956..621909877 100644 --- a/mypy-stubs/rdflib/paths.pyi +++ b/mypy-stubs/rdflib/paths.pyi @@ -1,5 +1,5 @@ -from collections.abc import Generator -from typing import Any, Callable, Union +from collections.abc import Callable, Generator +from typing import Any, Union from rdflib.term import Node as Node from rdflib.term import URIRef as URIRef diff --git a/mypy-stubs/rdflib/plugin.pyi b/mypy-stubs/rdflib/plugin.pyi index 9b94e99f3..cef006f0c 100644 --- a/mypy-stubs/rdflib/plugin.pyi +++ b/mypy-stubs/rdflib/plugin.pyi @@ -6,5 +6,5 @@ def register(name: str, kind: Any, module_path: str, class_name: str) -> None: . PluginT = TypeVar("PluginT") -def get(name: str, kind: Type[PluginT]) -> Type[PluginT]: ... +def get(name: str, kind: type[PluginT]) -> type[PluginT]: ... def plugins(name: Any | None = ..., kind: Any | None = ...) -> None: ... diff --git a/mypy-stubs/rdflib/query.pyi b/mypy-stubs/rdflib/query.pyi index 981fe12d2..a72bf711e 100644 --- a/mypy-stubs/rdflib/query.pyi +++ b/mypy-stubs/rdflib/query.pyi @@ -1,12 +1,12 @@ -from typing import IO, Any, Dict, Iterator, List, Mapping, Optional, Tuple, overload +from collections.abc import Iterator, Mapping +from typing import IO, Any, Dict, List, Optional, SupportsIndex, Tuple, overload from rdflib import URIRef, Variable from rdflib.term import Identifier -from typing_extensions import SupportsIndex -class ResultRow(Tuple["Identifier", ...]): +class ResultRow(tuple["Identifier", ...]): def __new__( - cls, values: Mapping[Variable, Identifier], labels: List[Variable] + cls, values: Mapping[Variable, Identifier], labels: list[Variable] ) -> ResultRow: ... def __getattr__(self, name: str) -> Identifier: ... @overload @@ -14,9 +14,9 @@ class ResultRow(Tuple["Identifier", ...]): @overload def __getitem__(self, __x: SupportsIndex) -> Identifier: ... @overload - def __getitem__(self, __x: slice) -> Tuple[Identifier, ...]: ... + def __getitem__(self, __x: slice) -> tuple[Identifier, ...]: ... def get(self, name: str, default: Any | None = ...) -> Identifier: ... - def asdict(self) -> Dict[str, Identifier]: ... + def asdict(self) -> dict[str, Identifier]: ... class Result: type: Any @@ -39,4 +39,4 @@ class Result: encoding: str = ..., format: str = ..., **args: Any, - ) -> Optional[bytes]: ... + ) -> bytes | None: ... diff --git a/mypy-stubs/rdflib/resource.pyi b/mypy-stubs/rdflib/resource.pyi index 0dd3b988e..d9e16dc55 100644 --- a/mypy-stubs/rdflib/resource.pyi +++ b/mypy-stubs/rdflib/resource.pyi @@ -1,4 +1,5 @@ -from typing import Any, Iterable, Iterator, Tuple +from collections.abc import Iterable, Iterator +from typing import Any, Tuple from _typeshed import Incomplete from rdflib.graph import Graph, Seq @@ -14,9 +15,9 @@ class Resource: def subjects(self, predicate: Any | None = ...) -> Iterable[Node]: ... def predicates(self, o: Incomplete | None = ...) -> Iterable[Node]: ... def objects(self, predicate: Any | None = ...) -> Iterable[Node]: ... - def subject_predicates(self) -> Iterator[Tuple[Node, Node]]: ... - def subject_objects(self) -> Iterator[Tuple[Node, Node]]: ... - def predicate_objects(self) -> Iterator[Tuple[Node, Node]]: ... + def subject_predicates(self) -> Iterator[tuple[Node, Node]]: ... + def subject_objects(self) -> Iterator[tuple[Node, Node]]: ... + def predicate_objects(self) -> Iterator[tuple[Node, Node]]: ... def value( self, p: Node, o: Node | None = ..., default: Any | None = ..., any: bool = ... ) -> Any: ... diff --git a/mypy-stubs/rdflib/term.pyi b/mypy-stubs/rdflib/term.pyi index 0830bdc29..83a595b54 100644 --- a/mypy-stubs/rdflib/term.pyi +++ b/mypy-stubs/rdflib/term.pyi @@ -1,9 +1,10 @@ -from typing import Any, Callable, Union +from collections.abc import Callable +from typing import Any, Union class Node: ... class Identifier(Node, str): - def __new__(cls, value: Union[Any, str, None]) -> "Identifier": ... + def __new__(cls, value: Any | str | None) -> "Identifier": ... def eq(self, other: Any) -> bool: ... def neq(self, other: Any) -> bool: ... diff --git a/mypy-stubs/spython/main/__init__.pyi b/mypy-stubs/spython/main/__init__.pyi index adced1f3a..ac22f83cb 100644 --- a/mypy-stubs/spython/main/__init__.pyi +++ b/mypy-stubs/spython/main/__init__.pyi @@ -1,4 +1,5 @@ -from typing import Iterator, Optional +from collections.abc import Iterator +from typing import Optional from .base import Client as _BaseClient from .build import build as base_build diff --git a/mypy-stubs/spython/main/build.pyi b/mypy-stubs/spython/main/build.pyi index 098ba3436..e2b295cc1 100644 --- a/mypy-stubs/spython/main/build.pyi +++ b/mypy-stubs/spython/main/build.pyi @@ -1,23 +1,24 @@ -from typing import Iterator, Optional +from collections.abc import Iterator +from typing import Optional from .base import Client def build( self: Client, - recipe: Optional[str] = ..., - image: Optional[str] = ..., - isolated: Optional[bool] = ..., - sandbox: Optional[bool] = ..., - writable: Optional[bool] = ..., - build_folder: Optional[str] = ..., - robot_name: Optional[bool] = ..., - ext: Optional[str] = ..., - sudo: Optional[bool] = ..., - stream: Optional[bool] = ..., - force: Optional[bool] = ..., - options: Optional[list[str]] | None = ..., - quiet: Optional[bool] = ..., - return_result: Optional[bool] = ..., - sudo_options: Optional[str | list[str]] = ..., - singularity_options: Optional[list[str]] = ..., + recipe: str | None = ..., + image: str | None = ..., + isolated: bool | None = ..., + sandbox: bool | None = ..., + writable: bool | None = ..., + build_folder: str | None = ..., + robot_name: bool | None = ..., + ext: str | None = ..., + sudo: bool | None = ..., + stream: bool | None = ..., + force: bool | None = ..., + options: list[str] | None | None = ..., + quiet: bool | None = ..., + return_result: bool | None = ..., + sudo_options: str | list[str] | None = ..., + singularity_options: list[str] | None = ..., ) -> tuple[str, Iterator[str]]: ... diff --git a/mypy-stubs/spython/main/parse/recipe.pyi b/mypy-stubs/spython/main/parse/recipe.pyi index dabd4ebc5..21378549b 100644 --- a/mypy-stubs/spython/main/parse/recipe.pyi +++ b/mypy-stubs/spython/main/parse/recipe.pyi @@ -1,19 +1,19 @@ from typing import Optional class Recipe: - cmd: Optional[str] + cmd: str | None comments: list[str] - entrypoint: Optional[str] + entrypoint: str | None environ: list[str] files: list[str] layer_files: dict[str, str] install: list[str] labels: list[str] ports: list[str] - test: Optional[str] + test: str | None volumes: list[str] - workdir: Optional[str] + workdir: str | None layer: int - fromHeader: Optional[str] - source: Optional[Recipe] - def __init__(self, recipe: Optional[Recipe] = ..., layer: int = ...) -> None: ... + fromHeader: str | None + source: Recipe | None + def __init__(self, recipe: Recipe | None = ..., layer: int = ...) -> None: ... diff --git a/mypy-stubs/spython/main/parse/writers/singularity.pyi b/mypy-stubs/spython/main/parse/writers/singularity.pyi index c80198461..b3011d7ea 100644 --- a/mypy-stubs/spython/main/parse/writers/singularity.pyi +++ b/mypy-stubs/spython/main/parse/writers/singularity.pyi @@ -5,6 +5,6 @@ from .base import WriterBase as WriterBase class SingularityWriter(WriterBase): name: str - def __init__(self, recipe: Optional[dict[str, Recipe]] = ...) -> None: ... + def __init__(self, recipe: dict[str, Recipe] | None = ...) -> None: ... def validate(self) -> None: ... def convert(self, runscript: str = ..., force: bool = ...) -> str: ... diff --git a/pyproject.toml b/pyproject.toml index d18a1af1b..11fd829bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,4 +34,4 @@ before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-d [tool.black] line-length = 100 -target-version = [ "py39" ] +target-version = [ "py310" ] diff --git a/setup.py b/setup.py index a6817bcab..df3d2e996 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import Extension, setup if TYPE_CHECKING: - from typing_extensions import TypeGuard + from typing import TypeGuard if os.name == "nt": warnings.warn( @@ -170,7 +170,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "pillow", # workaround for https://github.com/galaxyproject/galaxy/pull/20525 ], }, - python_requires=">=3.9, <3.15", + python_requires=">=3.10, <3.15", use_scm_version=True, setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<10"], test_suite="tests", @@ -196,7 +196,6 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index c87cf0ef7..da2f5d138 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,14 +6,14 @@ import json from io import StringIO -from typing import Any, Optional +from typing import Any from cwltest import utils def pytest_cwl_execute_test( - config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> tuple[int, Optional[dict[str, Any]]]: + config: utils.CWLTestConfig, processfile: str, jobfile: str | None +) -> tuple[int, dict[str, Any] | None]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 27dfae39d..93b1c7a47 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -287,7 +287,7 @@ def test_cuda_eval_resource_range() -> None: with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) - joborder = {} # type: CWLObjectType + joborder: CWLObjectType = {} loadingContext = LoadingContext({"do_update": True}) runtime_context = RuntimeContext({}) @@ -304,7 +304,7 @@ def test_cuda_eval_resource_max() -> None: with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) - joborder = {} # type: CWLObjectType + joborder: CWLObjectType = {} loadingContext = LoadingContext({"do_update": True}) runtime_context = RuntimeContext({}) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 046756f05..ac932ea94 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,6 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional import pytest @@ -16,7 +15,7 @@ from .util import get_data, get_main_output, get_tool_env, needs_docker -deps: Optional[ModuleType] = None +deps: ModuleType | None = None try: from galaxy.tool_util import deps except ImportError: @@ -58,7 +57,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[int | None, str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -110,7 +109,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[int | None, str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr diff --git a/tests/test_environment.py b/tests/test_environment.py index 6eb596631..b9fa24579 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,9 +2,9 @@ import os from abc import ABC, abstractmethod -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path -from typing import Callable, Union +from typing import Union import pytest from packaging.version import Version diff --git a/tests/test_examples.py b/tests/test_examples.py index 8eb8cf55d..f371f45ae 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Union, cast +from typing import Any, cast import cwl_utils.expression as expr import pydot @@ -403,8 +403,8 @@ def test_scandeps() -> None: } def loadref( - base: str, p: Union[CommentedMap, CommentedSeq, str, None] - ) -> Union[CommentedMap, CommentedSeq, str, None]: + base: str, p: CommentedMap | CommentedSeq | str | None + ) -> CommentedMap | CommentedSeq | str | None: if isinstance(p, dict): return p raise Exception("test case can't load things") @@ -508,8 +508,8 @@ def test_scandeps_samedirname() -> None: } def loadref( - base: str, p: Union[CommentedMap, CommentedSeq, str, None] - ) -> Union[CommentedMap, CommentedSeq, str, None]: + base: str, p: CommentedMap | CommentedSeq | str | None + ) -> CommentedMap | CommentedSeq | str | None: if isinstance(p, dict): return p raise Exception("test case can't load things") diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 962b7d7e5..55c4695fe 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, Optional +from typing import Any from urllib.parse import urljoin, urlsplit import pytest @@ -21,11 +21,11 @@ class CWLTestFetcher(Fetcher): def __init__( self, cache: CacheType, - session: Optional[requests.sessions.Session], + session: requests.sessions.Session | None, ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str: + def fetch_text(self, url: str, content_types: list[str] | None = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 @@ -36,7 +36,7 @@ def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str """ raise RuntimeError("Not foo.cwl, was %s" % url) - def check_exists(self, url): # type: (str) -> bool + def check_exists(self, url: str) -> bool: return url == "baz:bar/foo.cwl" def urljoin(self, base: str, url: str) -> str: diff --git a/tests/test_mpi.py b/tests/test_mpi.py index b36392a8a..9683ffa22 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -7,7 +7,7 @@ from importlib.resources import files from io import StringIO from pathlib import Path -from typing import Any, Optional, cast +from typing import Any, cast import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -306,8 +306,8 @@ def schema_ext11() -> Generator[Names, None, None]: def mk_tool( schema: Names, opts: list[str], - reqs: Optional[list[CommentedMap]] = None, - hints: Optional[list[CommentedMap]] = None, + reqs: list[CommentedMap] | None = None, + hints: list[CommentedMap] | None = None, ) -> tuple[RuntimeContext, CommandLineTool]: tool = basetool.copy() diff --git a/tests/test_secrets.py b/tests/test_secrets.py index a8c0b67af..206b001bd 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile +from collections.abc import Callable from io import StringIO -from typing import Callable, Union import pytest @@ -41,7 +41,7 @@ def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, list[str], dict[str, str]], + expected: str | list[str] | dict[str, str], secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index eb99124ea..fd5d22af0 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -225,6 +225,7 @@ def test_single_process_subwf_subwf_inline_step(tmp_path: Path) -> None: [ "--outdir", str(tmp_path), + "--debug", "--single-process", "step1/stepX/stepY", get_data("tests/subgraph/count-lines17-wf.cwl.json"), diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 2e50fe722..2bf7a809a 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse +from collections.abc import Callable from io import StringIO from pathlib import Path -from typing import Callable import pytest diff --git a/tests/test_trs.py b/tests/test_trs.py index 5f94ce58a..c51b6e12b 100644 --- a/tests/test_trs.py +++ b/tests/test_trs.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any from unittest import mock from unittest.mock import MagicMock @@ -10,7 +10,7 @@ class MockResponse1: def __init__( - self, json_data: Any, status_code: int, raise_for_status: Optional[bool] = None + self, json_data: Any, status_code: int, raise_for_status: bool | None = None ) -> None: """Create a fake return object for requests.Session.head.""" self.json_data = json_data @@ -28,7 +28,7 @@ def mocked_requests_head(*args: Any, **kwargs: Any) -> MockResponse1: class MockResponse2: def __init__( - self, json_data: Any, status_code: int, raise_for_status: Optional[bool] = None + self, json_data: Any, status_code: int, raise_for_status: bool | None = None ) -> None: """Create a fake return object for requests.Session.get.""" self.json_data = json_data diff --git a/tests/util.py b/tests/util.py index d7624bc5e..235c3e6c3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -12,7 +12,7 @@ from contextlib import ExitStack from importlib.resources import as_file, files from pathlib import Path -from typing import Any, Optional, Union +from typing import Any import pytest @@ -66,7 +66,7 @@ def get_data(filename: str) -> str: reason="Requires the podman executable on the system path.", ) -_env_accepts_null: Optional[bool] = None +_env_accepts_null: bool | None = None def env_accepts_null() -> bool: @@ -85,11 +85,11 @@ def env_accepts_null() -> bool: def get_main_output( args: list[str], - replacement_env: Optional[Mapping[str, str]] = None, - extra_env: Optional[Mapping[str, str]] = None, - monkeypatch: Optional[pytest.MonkeyPatch] = None, + replacement_env: Mapping[str, str] | None = None, + extra_env: Mapping[str, str] | None = None, + monkeypatch: pytest.MonkeyPatch | None = None, **extra_kwargs: Any, -) -> tuple[Optional[int], str, str]: +) -> tuple[int | None, str, str]: """Run cwltool main. args: the command line args to call it with @@ -130,11 +130,11 @@ def get_main_output( def get_tool_env( tmp_path: Path, flag_args: list[str], - inputs_file: Optional[str] = None, - replacement_env: Optional[Mapping[str, str]] = None, - extra_env: Optional[Mapping[str, str]] = None, - monkeypatch: Optional[pytest.MonkeyPatch] = None, - runtime_env_accepts_null: Optional[bool] = None, + inputs_file: str | None = None, + replacement_env: Mapping[str, str] | None = None, + extra_env: Mapping[str, str] | None = None, + monkeypatch: pytest.MonkeyPatch | None = None, + runtime_env_accepts_null: bool | None = None, ) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's @@ -169,7 +169,7 @@ def get_tool_env( @contextlib.contextmanager -def working_directory(path: Union[str, Path]) -> Generator[None, None, None]: +def working_directory(path: str | Path) -> Generator[None, None, None]: """Change working directory and returns to previous on exit.""" prev_cwd = Path.cwd() os.chdir(path) diff --git a/tox.ini b/tox.ini index 547d63abf..5350201cd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{9,10,11,12,13,14}-lint - py3{9,10,11,12,13,14}-unit - py3{9,10,11,12,13,14}-bandit - py3{9,10,11,12,13,14}-mypy + py3{10,11,12,13,14}-lint + py3{10,11,12,13,14}-unit + py3{10,11,12,13,14}-bandit + py3{10,11,12,13,14}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -16,7 +16,6 @@ testpaths = tests [gh-actions] python = - 3.9: py39 3.10: py310 3.11: py311 3.12: py312 @@ -25,13 +24,13 @@ python = [testenv] skipsdist = - py3{9,10,11,12,13,14}-!{unit,mypy,lintreadme} = True + py3{10,11,12,13,14}-!{unit,mypy,lintreadme} = True description = - py3{9,10,11,12,13,14}-unit: Run the unit tests - py3{9,10,11,12,13,14}-lint: Lint the Python code - py3{9,10,11,12,13,14}-bandit: Search for common security issues - py3{9,10,11,12,13,14}-mypy: Check for type safety + py3{10,11,12,13,14}-unit: Run the unit tests + py3{10,11,12,13,14}-lint: Lint the Python code + py3{10,11,12,13,14}-bandit: Search for common security issues + py3{10,11,12,13,14}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{9,10,11,12,13,14}-unit: deps + py3{10,11,12,13,14}-unit: deps deps = - py3{9,10,11,12,13,14}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{9,10,11,12,13,14}-{unit,mypy}: -rtest-requirements.txt - py3{9,10,11,12,13,14}-lint: -rlint-requirements.txt - py3{9,10,11,12,13,14}-bandit: bandit - py3{9,10,11,12,13,14}-mypy: -rmypy-requirements.txt + py3{10,11,12,13,14}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{10,11,12,13,14}-{unit,mypy}: -rtest-requirements.txt + py3{10,11,12,13,14}-lint: -rlint-requirements.txt + py3{10,11,12,13,14}-bandit: bandit + py3{10,11,12,13,14}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,19 +62,19 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{9,10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" - py3{9,10,11,12,13,14}-bandit: bandit -r cwltool - py3{9,10,11,12,13,14}-lint: make flake8 format-check codespell-check - py3{9,10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" + py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" + py3{10,11,12,13,14}-bandit: bandit -r cwltool + py3{10,11,12,13,14}-lint: make flake8 format-check codespell-check + py3{10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{9,10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make