Parallel concatenate (#5926)

* Simplify concatenate

* First attempt at parallel concatenate

* Clean up a bit

* Add support for comparing different data types

* Undo unnessary change

* More tests

* Use faster lookup

* Add test to show that NaNs are considered equal

* Avoid inserting closures into the Dask graph

* Fix type hints

* Compute numpy array hashes immediately

* Concatenate 25 cubes instead of 2

* Improve test coverage

* Add whatsnew entry

* Various improvements from review

* Use correct value for chunks for numpy arrays

* Python 3.10 compatibility

* Avoid creating derived coordinates multiple times

* Support comparing differently shaped arrays

* Rewrite for code style without multiple returns

* Remove print call

* Better hashing algorithm

* Add more information to release notes

Author: bouweandela

benchmarks/benchmarks/merge_concat.py

@@ -4,9 +4,12 @@
 # See LICENSE in the root of the repository for full licensing details.
 """Benchmarks relating to :meth:`iris.cube.CubeList.merge` and ``concatenate``."""
 
+import warnings
+
 import numpy as np
 
 from iris.cube import CubeList
+from iris.warnings import IrisVagueMetadataWarning
 
 from .generate_data.stock import realistic_4d_w_everything
 
@@ -44,19 +47,26 @@ class Concatenate:
 
     cube_list: CubeList
 
-    def setup(self):
-        source_cube = realistic_4d_w_everything()
-        second_cube = source_cube.copy()
-        first_dim_coord = second_cube.coord(dimensions=0, dim_coords=True)
-        first_dim_coord.points = (
-            first_dim_coord.points + np.ptp(first_dim_coord.points) + 1
-        )
-        self.cube_list = CubeList([source_cube, second_cube])
-
-    def time_concatenate(self):
+    params = [[False, True]]
+    param_names = ["Lazy operations"]
+
+    def setup(self, lazy_run: bool):
+        warnings.filterwarnings("ignore", message="Ignoring a datum")
+        warnings.filterwarnings("ignore", category=IrisVagueMetadataWarning)
+        source_cube = realistic_4d_w_everything(lazy=lazy_run)
+        self.cube_list = CubeList([source_cube])
+        for _ in range(24):
+            next_cube = self.cube_list[-1].copy()
+            first_dim_coord = next_cube.coord(dimensions=0, dim_coords=True)
+            first_dim_coord.points = (
+                first_dim_coord.points + np.ptp(first_dim_coord.points) + 1
+            )
+            self.cube_list.append(next_cube)
+
+    def time_concatenate(self, _):
         _ = self.cube_list.concatenate_cube()
 
-    def tracemalloc_concatenate(self):
+    def tracemalloc_concatenate(self, _):
         _ = self.cube_list.concatenate_cube()
 
     tracemalloc_concatenate.number = 3  # type: ignore[attr-defined]

docs/src/whatsnew/latest.rst

@@ -57,6 +57,18 @@ This document explains the changes made to Iris for this release
    with cftime :class:`~cftime.datetime` objects can benefit from the same
    improvement by adding a type hint to their category funcion. (:pull:`5999`)
 
+#. `@bouweandela`_ made :meth:`iris.cube.CubeList.concatenate` faster if more
+   than two cubes are concatenated with equality checks on the values of
+   auxiliary coordinates, derived coordinates, cell measures, or ancillary
+   variables enabled.
+   In some cases, this may lead to higher memory use. This can be remedied by
+   reducing the number of Dask workers.
+   In rare cases, the new implementation could potentially be slower. This
+   may happen when there are very many or large auxiliary coordinates, derived
+   coordinates, cell measures, or ancillary variables to be checked that span
+   the concatenation axis. This issue can be avoided by disabling the
+   problematic check. (:pull:`5926`)
+
 🔥 Deprecations
 ===============