From 8497d91bc3d8489501bd9291a9d1bc114bbc5737 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 12:36:26 +0200 Subject: [PATCH 01/23] optional import versions to pycompat --- xarray/core/dask_array_compat.py | 11 +++++------ xarray/core/indexing.py | 10 ++-------- xarray/core/pycompat.py | 11 ++++++++++- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/xarray/core/dask_array_compat.py b/xarray/core/dask_array_compat.py index 9f24590af8e..c0b99d430d4 100644 --- a/xarray/core/dask_array_compat.py +++ b/xarray/core/dask_array_compat.py @@ -1,13 +1,12 @@ import warnings -from distutils.version import LooseVersion import numpy as np +from .pycompat import dask_version + try: import dask.array as da - from dask import __version__ as dask_version except ImportError: - dask_version = "0.0.0" da = None @@ -57,7 +56,7 @@ def pad(array, pad_width, mode="constant", **kwargs): return padded -if LooseVersion(dask_version) > LooseVersion("2.30.0"): +if dask_version > "2.30.0": ensure_minimum_chunksize = da.overlap.ensure_minimum_chunksize else: @@ -114,7 +113,7 @@ def ensure_minimum_chunksize(size, chunks): return tuple(output) -if LooseVersion(dask_version) > LooseVersion("2021.03.0"): +if dask_version > "2021.03.0": sliding_window_view = da.lib.stride_tricks.sliding_window_view else: @@ -180,7 +179,7 @@ def sliding_window_view(x, window_shape, axis=None): axis=axis, ) # map_overlap's signature changed in https://github.com/dask/dask/pull/6165 - if LooseVersion(dask_version) > "2.18.0": + if dask_version > "2.18.0": return map_overlap(_np_sliding_window_view, x, align_arrays=False, **kwargs) else: return map_overlap(x, _np_sliding_window_view, **kwargs) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 76a0c6888b2..81e4264a0c5 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -8,16 +8,10 @@ import numpy as np import pandas as pd -try: - import dask - - DASK_VERSION = LooseVersion(dask.__version__) -except ModuleNotFoundError: - DASK_VERSION = LooseVersion("0") - from . import duck_array_ops, nputils, utils from .pycompat import ( dask_array_type, + dask_version, integer_types, is_duck_dask_array, sparse_array_type, @@ -1393,7 +1387,7 @@ def __getitem__(self, key): return value def __setitem__(self, key, value): - if DASK_VERSION >= "2021.04.1": + if dask_version >= "2021.04.1": if isinstance(key, BasicIndexer): self.array[key.tuple] = value elif isinstance(key, VectorizedIndexer): diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 8d613038957..19801536b61 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -1,3 +1,5 @@ +from distutils.version import LooseVersion + import numpy as np from .utils import is_duck_array @@ -5,9 +7,11 @@ integer_types = (int, np.integer) try: - import dask.array + import dask from dask.base import is_dask_collection + dask_version = LooseVersion(dask.__version__) + # solely for isinstance checks dask_array_type = (dask.array.Array,) @@ -16,6 +20,7 @@ def is_duck_dask_array(x): except ImportError: # pragma: no cover + dask_version = LooseVersion("0.0.0") dask_array_type = () is_duck_dask_array = lambda _: False is_dask_collection = lambda _: False @@ -24,14 +29,18 @@ def is_duck_dask_array(x): # solely for isinstance checks import sparse + sparse_version = LooseVersion(sparse.__version__) sparse_array_type = (sparse.SparseArray,) except ImportError: # pragma: no cover + sparse_version = LooseVersion("0.0.0") sparse_array_type = () try: # solely for isinstance checks import cupy + cupy_version = LooseVersion(sparse.__version__) cupy_array_type = (cupy.ndarray,) except ImportError: # pragma: no cover + cupy_version = LooseVersion("0.0.0") cupy_array_type = () From 2c119485b95a9aa6c80521b4a2f23f2546779fdb Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 12:44:14 +0200 Subject: [PATCH 02/23] Update indexing.py --- xarray/core/indexing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 81e4264a0c5..cd8dcb9ad3c 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -2,7 +2,6 @@ import functools import operator from collections import defaultdict -from distutils.version import LooseVersion from typing import Any, Callable, Iterable, List, Sequence, Tuple, Union import numpy as np From aeea5ccd934d6781fbc2dae6d0d331ba18284e86 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 13:02:37 +0200 Subject: [PATCH 03/23] move dtypes to avoid circular import --- xarray/core/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 62b66278b24..98d558008f2 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -31,8 +31,6 @@ import numpy as np import pandas as pd -from . import dtypes - K = TypeVar("K") V = TypeVar("V") T = TypeVar("T") @@ -83,9 +81,10 @@ def maybe_coerce_to_str(index, original_coords): pd.Index uses object-dtype to store str - try to avoid this for coords """ + from .dtypes import result_type as dtypes_result_type try: - result_type = dtypes.result_type(*original_coords) + result_type = dtypes_result_type(*original_coords) except TypeError: pass else: From 8fa535c3ab7c4a6dd8fc2fd9db04450bb8a5f0ab Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 13:11:29 +0200 Subject: [PATCH 04/23] Update pycompat.py --- xarray/core/pycompat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 19801536b61..fbe53e2a435 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -8,6 +8,7 @@ try: import dask + import dask.array from dask.base import is_dask_collection dask_version = LooseVersion(dask.__version__) From ec6c301b3a87e81165b0a5ea1e6751a7778babb1 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 13:39:54 +0200 Subject: [PATCH 05/23] faster unstacking --- xarray/core/dataset.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index abc4a0795d6..5e6992cfec6 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -78,7 +78,12 @@ ) from .missing import get_clean_interp_index from .options import OPTIONS, _get_keep_attrs -from .pycompat import is_duck_dask_array, sparse_array_type +from .pycompat import ( + dask_version, + is_duck_dask_array, + sparse_array_type, + sparse_version, +) from .utils import ( Default, Frozen, @@ -4030,18 +4035,12 @@ def unstack( for dim in dims: if ( - # Dask arrays don't support assignment by index, which the fast unstack - # function requires. - # https://github.com/pydata/xarray/pull/4746#issuecomment-753282125 - any(is_duck_dask_array(v.data) for v in self.variables.values()) - # Sparse doesn't currently support (though we could special-case - # it) + # Dask arrays supports assignment by index, + # https://github.com/dask/dask/pull/7393 + dask_version < "2021.04.0" + # Sparse now supports kwargs in full_like, # https://github.com/pydata/sparse/issues/422 - or any( - isinstance(v.data, sparse_array_type) - for v in self.variables.values() - ) - or sparse + or sparse_version < "0.11.2" # Until https://github.com/pydata/xarray/pull/4751 is resolved, # we check explicitly whether it's a numpy array. Once that is # resolved, explicitly exclude pint arrays. From 3f2784d841c6c6dd067f61abbeba0f9a144e3480 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 13:43:50 +0200 Subject: [PATCH 06/23] Update dataset.py --- xarray/core/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 5e6992cfec6..41b3ef27c66 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -81,7 +81,6 @@ from .pycompat import ( dask_version, is_duck_dask_array, - sparse_array_type, sparse_version, ) from .utils import ( From de15c39c392c5ad69d51c7eba978086c28cf4b39 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 13:48:48 +0200 Subject: [PATCH 07/23] Update dataset.py --- xarray/core/dataset.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 41b3ef27c66..da862f6ee77 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -78,11 +78,7 @@ ) from .missing import get_clean_interp_index from .options import OPTIONS, _get_keep_attrs -from .pycompat import ( - dask_version, - is_duck_dask_array, - sparse_version, -) +from .pycompat import dask_version, is_duck_dask_array, sparse_version from .utils import ( Default, Frozen, From 682922fda64455086b019d3981fd6446941f067d Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 15:17:56 +0200 Subject: [PATCH 08/23] have to check that the array type is in use --- xarray/core/dataset.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index da862f6ee77..ff538d2ffec 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -78,7 +78,12 @@ ) from .missing import get_clean_interp_index from .options import OPTIONS, _get_keep_attrs -from .pycompat import dask_version, is_duck_dask_array, sparse_version +from .pycompat import ( + dask_version, + is_duck_dask_array, + sparse_array_type, + sparse_version +) from .utils import ( Default, Frozen, @@ -4032,10 +4037,22 @@ def unstack( if ( # Dask arrays supports assignment by index, # https://github.com/dask/dask/pull/7393 - dask_version < "2021.04.0" + ( + dask_version < "2021.04.0" + and any(is_duck_dask_array(v.data) for v in self.variables.values()) + ) # Sparse now supports kwargs in full_like, # https://github.com/pydata/sparse/issues/422 - or sparse_version < "0.11.2" + or ( + sparse_version < "0.11.2" + and ( + any( + isinstance(v.data, sparse_array_type) + for v in self.variables.values() + ) + or sparse + ) + ) # Until https://github.com/pydata/xarray/pull/4751 is resolved, # we check explicitly whether it's a numpy array. Once that is # resolved, explicitly exclude pint arrays. From dba4693d3e7cf0a9437f1ce34526e6741dd8b1d1 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 15:22:18 +0200 Subject: [PATCH 09/23] Update dataset.py --- xarray/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index ff538d2ffec..c5a45a91279 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -82,7 +82,7 @@ dask_version, is_duck_dask_array, sparse_array_type, - sparse_version + sparse_version, ) from .utils import ( Default, From bb74dc9b3da0021a8268d6ccdbd960781c461f4c Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 16:54:09 +0200 Subject: [PATCH 10/23] sparse arg requires the slow path? --- xarray/core/dataset.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index c5a45a91279..39929587c27 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4045,14 +4045,13 @@ def unstack( # https://github.com/pydata/sparse/issues/422 or ( sparse_version < "0.11.2" - and ( - any( - isinstance(v.data, sparse_array_type) - for v in self.variables.values() - ) - or sparse + and any( + isinstance(v.data, sparse_array_type) + for v in self.variables.values() ) ) + # Shifting the arrays to sparse requires _unstack_full_reindex: + or sparse # Until https://github.com/pydata/xarray/pull/4751 is resolved, # we check explicitly whether it's a numpy array. Once that is # resolved, explicitly exclude pint arrays. From 8f15a28ed9743437e0be97a7c4015479fce0a420 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 18:19:29 +0200 Subject: [PATCH 11/23] cupy.__version__ --- xarray/core/pycompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index fbe53e2a435..9f47da6c8cc 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -40,7 +40,7 @@ def is_duck_dask_array(x): # solely for isinstance checks import cupy - cupy_version = LooseVersion(sparse.__version__) + cupy_version = LooseVersion(cupy.__version__) cupy_array_type = (cupy.ndarray,) except ImportError: # pragma: no cover cupy_version = LooseVersion("0.0.0") From c2ce0cf4b0c1227a7c309bfdc90b65cd1c409e66 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 21:57:36 +0200 Subject: [PATCH 12/23] test pint_array_type --- xarray/core/dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 39929587c27..7f92699ccff 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4058,12 +4058,12 @@ def unstack( # # pint doesn't implement `np.full_like` in a way that's # # currently compatible. # # https://github.com/pydata/xarray/pull/4746#issuecomment-753425173 - # # or any( - # # isinstance(v.data, pint_array_type) for v in self.variables.values() - # # ) or any( - not isinstance(v.data, np.ndarray) for v in self.variables.values() + isinstance(v.data, pint_array_type) for v in self.variables.values() ) + # or any( + # not isinstance(v.data, np.ndarray) for v in self.variables.values() + # ) ): result = result._unstack_full_reindex(dim, fill_value, sparse) else: From 9992824b59701ef32f3b3495ce71cd673c8268da Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 22:01:21 +0200 Subject: [PATCH 13/23] Update dataset.py --- xarray/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 7f92699ccff..722d4032713 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4062,7 +4062,7 @@ def unstack( isinstance(v.data, pint_array_type) for v in self.variables.values() ) # or any( - # not isinstance(v.data, np.ndarray) for v in self.variables.values() + # not isinstance(v.data, np.ndarray) for v in self.variables.values() # ) ): result = result._unstack_full_reindex(dim, fill_value, sparse) From 95fc10e1962ebc1925bb1a6a761d9bb45fe1b4ea Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 22:15:25 +0200 Subject: [PATCH 14/23] Add pint check in pycompat --- xarray/core/dataset.py | 1 + xarray/core/pycompat.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 722d4032713..5b85c098b75 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -81,6 +81,7 @@ from .pycompat import ( dask_version, is_duck_dask_array, + pint_array_type, sparse_array_type, sparse_version, ) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 9f47da6c8cc..f8c554fac26 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -45,3 +45,12 @@ def is_duck_dask_array(x): except ImportError: # pragma: no cover cupy_version = LooseVersion("0.0.0") cupy_array_type = () + +try: + import pint + + pint_version = LooseVersion(pint.__version__) + pint_array_type = (pint.Quantity,) +except ImportError: # pragma: no cover + pint_version = LooseVersion("0.0.0") + pint_array_type = () \ No newline at end of file From 2597619379417129fc41299f64c30d6c6d5277a9 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 22:17:19 +0200 Subject: [PATCH 15/23] Update pycompat.py --- xarray/core/pycompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index f8c554fac26..065e656a070 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -53,4 +53,4 @@ def is_duck_dask_array(x): pint_array_type = (pint.Quantity,) except ImportError: # pragma: no cover pint_version = LooseVersion("0.0.0") - pint_array_type = () \ No newline at end of file + pint_array_type = () From cf803600ea5101da8a6177e9d4a3dfd5b30634dd Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sat, 15 May 2021 22:52:21 +0200 Subject: [PATCH 16/23] revert pint test --- xarray/core/dataset.py | 9 ++++----- xarray/core/pycompat.py | 9 --------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 5b85c098b75..170e50462ad 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -81,7 +81,6 @@ from .pycompat import ( dask_version, is_duck_dask_array, - pint_array_type, sparse_array_type, sparse_version, ) @@ -4059,12 +4058,12 @@ def unstack( # # pint doesn't implement `np.full_like` in a way that's # # currently compatible. # # https://github.com/pydata/xarray/pull/4746#issuecomment-753425173 - or any( - isinstance(v.data, pint_array_type) for v in self.variables.values() - ) # or any( - # not isinstance(v.data, np.ndarray) for v in self.variables.values() + # isinstance(v.data, pint_array_type) for v in self.variables.values() # ) + or any( + not isinstance(v.data, np.ndarray) for v in self.variables.values() + ) ): result = result._unstack_full_reindex(dim, fill_value, sparse) else: diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 065e656a070..9f47da6c8cc 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -45,12 +45,3 @@ def is_duck_dask_array(x): except ImportError: # pragma: no cover cupy_version = LooseVersion("0.0.0") cupy_array_type = () - -try: - import pint - - pint_version = LooseVersion(pint.__version__) - pint_array_type = (pint.Quantity,) -except ImportError: # pragma: no cover - pint_version = LooseVersion("0.0.0") - pint_array_type = () From b41c11322fd9dd89beb57024c703c4724e32bc8c Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 16 May 2021 06:36:18 +0200 Subject: [PATCH 17/23] import whole dtypes --- xarray/core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 98d558008f2..52621f3d922 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -81,10 +81,10 @@ def maybe_coerce_to_str(index, original_coords): pd.Index uses object-dtype to store str - try to avoid this for coords """ - from .dtypes import result_type as dtypes_result_type + from . import dtypes try: - result_type = dtypes_result_type(*original_coords) + result_type = dtypes.result_type(*original_coords) except TypeError: pass else: From a31754b4eea1e977aa1e9d2f49b41aa518d66c78 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 16 May 2021 09:57:38 +0200 Subject: [PATCH 18/23] Test turning off the ndarray check --- xarray/core/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 170e50462ad..6a0300734bc 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4061,9 +4061,9 @@ def unstack( # or any( # isinstance(v.data, pint_array_type) for v in self.variables.values() # ) - or any( - not isinstance(v.data, np.ndarray) for v in self.variables.values() - ) + # or any( + # not isinstance(v.data, np.ndarray) for v in self.variables.values() + # ) ): result = result._unstack_full_reindex(dim, fill_value, sparse) else: From 207609b78b4555489d4534c139bb24bb86f2a2ed Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 16 May 2021 23:31:16 +0200 Subject: [PATCH 19/23] sparse and pint doesn't work. Switch to a restrictive if --- xarray/core/dataset.py | 39 ++++++++------------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 6a0300734bc..69a617188a6 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -81,8 +81,6 @@ from .pycompat import ( dask_version, is_duck_dask_array, - sparse_array_type, - sparse_version, ) from .utils import ( Default, @@ -4033,41 +4031,20 @@ def unstack( result = self.copy(deep=False) for dim in dims: - if ( + not sparse # Dask arrays supports assignment by index, # https://github.com/dask/dask/pull/7393 - ( - dask_version < "2021.04.0" - and any(is_duck_dask_array(v.data) for v in self.variables.values()) - ) - # Sparse now supports kwargs in full_like, - # https://github.com/pydata/sparse/issues/422 - or ( - sparse_version < "0.11.2" - and any( - isinstance(v.data, sparse_array_type) - for v in self.variables.values() - ) + and all( + dask_version >= "2021.04.0" + and is_duck_dask_array(v.data) + or isinstance(v.data, np.ndarray) + for v in self.variables.values() ) - # Shifting the arrays to sparse requires _unstack_full_reindex: - or sparse - # Until https://github.com/pydata/xarray/pull/4751 is resolved, - # we check explicitly whether it's a numpy array. Once that is - # resolved, explicitly exclude pint arrays. - # # pint doesn't implement `np.full_like` in a way that's - # # currently compatible. - # # https://github.com/pydata/xarray/pull/4746#issuecomment-753425173 - # or any( - # isinstance(v.data, pint_array_type) for v in self.variables.values() - # ) - # or any( - # not isinstance(v.data, np.ndarray) for v in self.variables.values() - # ) ): - result = result._unstack_full_reindex(dim, fill_value, sparse) - else: result = result._unstack_once(dim, fill_value) + else: + result = result._unstack_full_reindex(dim, fill_value, sparse) return result def update(self, other: "CoercibleMapping") -> "Dataset": From 7bb5aa57c54934d64c8ec86fb057a526fed87453 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Sun, 16 May 2021 23:35:37 +0200 Subject: [PATCH 20/23] Update dataset.py --- xarray/core/dataset.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 69a617188a6..b7e78bb4219 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -78,10 +78,7 @@ ) from .missing import get_clean_interp_index from .options import OPTIONS, _get_keep_attrs -from .pycompat import ( - dask_version, - is_duck_dask_array, -) +from .pycompat import dask_version, is_duck_dask_array from .utils import ( Default, Frozen, From d5b31aeef288bfbc6670baa05a2d95a35c0b79a2 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 17 May 2021 23:22:40 +0200 Subject: [PATCH 21/23] Add back some comments and add some relevant issue links --- xarray/core/dataset.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index a9ca1d82ae2..c61eb90995b 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4030,17 +4030,25 @@ def unstack( for dim in dims: if ( not sparse - # Dask arrays supports assignment by index, - # https://github.com/dask/dask/pull/7393 and all( + # Dask arrays recently supports assignment by index, + # https://github.com/dask/dask/pull/7393 dask_version >= "2021.04.0" and is_duck_dask_array(v.data) + # Numpy arrays handles the fast path: or isinstance(v.data, np.ndarray) for v in self.variables.values() ) ): + # Fast unstacking path: result = result._unstack_once(dim, fill_value) else: + # Slower unstacking path, examples of array types that + # currently has to use this path: + # * sparse doesn't support item assigment, + # https://github.com/pydata/sparse/issues/114 + # * pint has some circular import issues, + # https://github.com/pydata/xarray/pull/4751 result = result._unstack_full_reindex(dim, fill_value, sparse) return result From 35db4a89d7cc9e6ea81d5780da7c3272787c52e2 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 17 May 2021 23:29:21 +0200 Subject: [PATCH 22/23] lint --- xarray/core/dataset.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index c61eb90995b..d4797f495b3 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4028,17 +4028,14 @@ def unstack( result = self.copy(deep=False) for dim in dims: - if ( - not sparse - and all( - # Dask arrays recently supports assignment by index, - # https://github.com/dask/dask/pull/7393 - dask_version >= "2021.04.0" - and is_duck_dask_array(v.data) - # Numpy arrays handles the fast path: - or isinstance(v.data, np.ndarray) - for v in self.variables.values() - ) + if not sparse and all( + # Dask arrays recently supports assignment by index, + # https://github.com/dask/dask/pull/7393 + dask_version >= "2021.04.0" + and is_duck_dask_array(v.data) + # Numpy arrays handles the fast path: + or isinstance(v.data, np.ndarray) + for v in self.variables.values() ): # Fast unstacking path: result = result._unstack_once(dim, fill_value) From 7ee325d928d173a70ea55dda60bd7cef965b4073 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 17 May 2021 23:31:45 +0200 Subject: [PATCH 23/23] Update dataset.py --- xarray/core/dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index d4797f495b3..a42d045705c 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -4031,8 +4031,7 @@ def unstack( if not sparse and all( # Dask arrays recently supports assignment by index, # https://github.com/dask/dask/pull/7393 - dask_version >= "2021.04.0" - and is_duck_dask_array(v.data) + dask_version >= "2021.04.0" and is_duck_dask_array(v.data) # Numpy arrays handles the fast path: or isinstance(v.data, np.ndarray) for v in self.variables.values()