Merge pull request #5361 from SciTools/v3.6.x

v3.6.x merge-back

Author: trexfeathers

docs/src/whatsnew/3.6.rst

@@ -55,6 +55,56 @@ This document explains the changes made to Iris for this release
     or feature requests for improving Iris. Enjoy!
 
 
+v3.6.1 (26 June 2023)
+=====================
+
+.. dropdown:: v3.6.1 Patches
+   :color: primary
+   :icon: alert
+   :animate: fade-in
+
+   📢 **Announcements**
+
+   Welcome and congratulations to `@sloosvel`_ who made their first contribution to
+   Iris! 🎉
+
+   The patches in this release of Iris include:
+
+   ✨ **Features**
+
+   #. `@rcomer`_ rewrote :func:`~iris.util.broadcast_to_shape` so it now handles
+      lazy data. This pull-request has been included to support :pull:`5341`.
+      (:pull:`5307`) [``pre-v3.7.0``]
+
+   🐛 **Bugs Fixed**
+
+   #. `@stephenworsley`_ fixed :meth:`~iris.cube.Cube.convert_units` to allow unit
+      conversion of lazy data when using a `Distributed`_ scheduler.
+      (:issue:`5347`, :pull:`5349`)
+
+   #. `@schlunma`_ fixed a bug in the concatenation of cubes with aux factories
+      which could lead to a `KeyError` due to dependencies that have not been
+      properly updated.
+      (:issue:`5339`, :pull:`5340`)
+
+   #. `@schlunma`_ fixed a bug which realized all weights during weighted
+      aggregation. Now weighted aggregation is fully lazy again.
+      (:issue:`5338`, :pull:`5341`)
+
+   🚀 **Performance Enhancements**
+
+   #. `@sloosvel`_ improved :meth:`~iris.cube.CubeList.concatenate_cube` and
+      :meth:`~iris.cube.CubeList.concatenate` to ensure that lazy auxiliary coordinate
+      points and bounds are not realized. This change now allows cubes with
+      high-resolution auxiliary coordinates to concatenate successfully whilst using a
+      minimal in-core memory footprint.
+      (:issue:`5115`, :pull:`5142`)
+
+   Note that, the above contribution labelled with ``pre-v3.7.0`` is part of the
+   forthcoming Iris ``v3.7.0`` release, but requires to be included in this patch
+   release.
+
+
 📢 Announcements
 ================
 
@@ -169,6 +219,7 @@ This document explains the changes made to Iris for this release
     core dev names are automatically included by the common_links.inc:
 
 .. _@fnattino: https://github.com/fnattino
+.. _@sloosvel: https://github.com/sloosvel
 
 
 .. comment
@@ -180,4 +231,5 @@ This document explains the changes made to Iris for this release
 .. _PEP-0621: https://peps.python.org/pep-0621/
 .. _pypa/build: https://pypa-build.readthedocs.io/en/stable/
 .. _NEP29: https://numpy.org/neps/nep-0029-deprecation_policy.html
-.. _Contributor Covenant: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
+.. _Contributor Covenant: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
+.. _Distributed: https://distributed.dask.org/en/stable/

lib/iris/_concatenate.py

@@ -101,11 +101,15 @@ def __new__(mcs, coord, dims):
 
         """
         defn = coord.metadata
-        points_dtype = coord.points.dtype
-        bounds_dtype = coord.bounds.dtype if coord.bounds is not None else None
+        points_dtype = coord.core_points().dtype
+        bounds_dtype = (
+            coord.core_bounds().dtype
+            if coord.core_bounds() is not None
+            else None
+        )
         kwargs = {}
         # Add scalar flag metadata.
-        kwargs["scalar"] = coord.points.size == 1
+        kwargs["scalar"] = coord.core_points().size == 1
         # Add circular flag metadata for dimensional coordinates.
         if hasattr(coord, "circular"):
             kwargs["circular"] = coord.circular
@@ -700,18 +704,27 @@ def _cmp(coord, other):
 
         """
         # A candidate axis must have non-identical coordinate points.
-        candidate_axis = not array_equal(coord.points, other.points)
+        candidate_axis = not array_equal(
+            coord.core_points(), other.core_points()
+        )
 
         if candidate_axis:
             # Ensure both have equal availability of bounds.
-            result = (coord.bounds is None) == (other.bounds is None)
+            result = (coord.core_bounds() is None) == (
+                other.core_bounds() is None
+            )
         else:
-            if coord.bounds is not None and other.bounds is not None:
+            if (
+                coord.core_bounds() is not None
+                and other.core_bounds() is not None
+            ):
                 # Ensure equality of bounds.
-                result = array_equal(coord.bounds, other.bounds)
+                result = array_equal(coord.core_bounds(), other.core_bounds())
             else:
                 # Ensure both have equal availability of bounds.
-                result = coord.bounds is None and other.bounds is None
+                result = (
+                    coord.core_bounds() is None and other.core_bounds() is None
+                )
 
         return result, candidate_axis
 
@@ -851,9 +864,13 @@ def concatenate(self):
             # Concatenate the new dimension coordinate.
             dim_coords_and_dims = self._build_dim_coordinates()
 
-            # Concatenate the new auxiliary coordinates.
+            # Concatenate the new auxiliary coordinates (does NOT include
+            # scalar coordinates!).
             aux_coords_and_dims = self._build_aux_coordinates()
 
+            # Concatenate the new scalar coordinates.
+            scalar_coords = self._build_scalar_coordinates()
+
             # Concatenate the new cell measures
             cell_measures_and_dims = self._build_cell_measures()
 
@@ -862,18 +879,21 @@ def concatenate(self):
 
             # Concatenate the new aux factories
             aux_factories = self._build_aux_factories(
-                dim_coords_and_dims, aux_coords_and_dims
+                dim_coords_and_dims, aux_coords_and_dims, scalar_coords
             )
 
             # Concatenate the new data payload.
             data = self._build_data()
 
             # Build the new cube.
+            all_aux_coords_and_dims = aux_coords_and_dims + [
+                (scalar_coord, ()) for scalar_coord in scalar_coords
+            ]
             kwargs = cube_signature.defn._asdict()
             cube = iris.cube.Cube(
                 data,
                 dim_coords_and_dims=dim_coords_and_dims,
-                aux_coords_and_dims=aux_coords_and_dims,
+                aux_coords_and_dims=all_aux_coords_and_dims,
                 cell_measures_and_dims=cell_measures_and_dims,
                 ancillary_variables_and_dims=ancillary_variables_and_dims,
                 aux_factories=aux_factories,
@@ -1095,7 +1115,7 @@ def _build_aux_coordinates(self):
                 # Concatenate the points together.
                 dim = dims.index(self.axis)
                 points = [
-                    skton.signature.aux_coords_and_dims[i].coord.points
+                    skton.signature.aux_coords_and_dims[i].coord.core_points()
                     for skton in skeletons
                 ]
                 points = np.concatenate(tuple(points), axis=dim)
@@ -1104,7 +1124,9 @@ def _build_aux_coordinates(self):
                 bnds = None
                 if coord.has_bounds():
                     bnds = [
-                        skton.signature.aux_coords_and_dims[i].coord.bounds
+                        skton.signature.aux_coords_and_dims[
+                            i
+                        ].coord.core_bounds()
                         for skton in skeletons
                     ]
                     bnds = np.concatenate(tuple(bnds), axis=dim)
@@ -1132,12 +1154,22 @@ def _build_aux_coordinates(self):
 
             aux_coords_and_dims.append((coord.copy(), dims))
 
-        # Generate all the scalar coordinates for the new concatenated cube.
-        for coord in cube_signature.scalar_coords:
-            aux_coords_and_dims.append((coord.copy(), ()))
-
         return aux_coords_and_dims
 
+    def _build_scalar_coordinates(self):
+        """
+        Generate the scalar coordinates for the new concatenated cube.
+
+        Returns:
+            A list of scalar coordinates.
+
+        """
+        scalar_coords = []
+        for coord in self._cube_signature.scalar_coords:
+            scalar_coords.append(coord.copy())
+
+        return scalar_coords
+
     def _build_cell_measures(self):
         """
         Generate the cell measures with associated dimension(s)
@@ -1216,7 +1248,9 @@ def _build_ancillary_variables(self):
 
         return ancillary_variables_and_dims
 
-    def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
+    def _build_aux_factories(
+        self, dim_coords_and_dims, aux_coords_and_dims, scalar_coords
+    ):
         """
         Generate the aux factories for the new concatenated cube.
 
@@ -1230,6 +1264,9 @@ def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
             A list of auxiliary coordinates and dimension(s) tuple pairs from
             the concatenated cube.
 
+        * scalar_coords:
+            A list of scalar coordinates from the concatenated cube.
+
         Returns:
             A list of :class:`iris.aux_factory.AuxCoordFactory`.
 
@@ -1240,35 +1277,44 @@ def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
         old_aux_coords = [a[0] for a in cube_signature.aux_coords_and_dims]
         new_dim_coords = [d[0] for d in dim_coords_and_dims]
         new_aux_coords = [a[0] for a in aux_coords_and_dims]
-        scalar_coords = cube_signature.scalar_coords
+        old_scalar_coords = cube_signature.scalar_coords
+        new_scalar_coords = scalar_coords
 
         aux_factories = []
 
         # Generate all the factories for the new concatenated cube.
-        for i, (coord, dims, factory) in enumerate(
-            cube_signature.derived_coords_and_dims
-        ):
-            # Check whether the derived coordinate of the factory spans the
-            # nominated dimension of concatenation.
-            if self.axis in dims:
-                # Update the dependencies of the factory with coordinates of
-                # the concatenated cube. We need to check all coordinate types
-                # here (dim coords, aux coords, and scalar coords).
-                new_dependencies = {}
-                for old_dependency in factory.dependencies.values():
-                    if old_dependency in old_dim_coords:
-                        dep_idx = old_dim_coords.index(old_dependency)
-                        new_dependency = new_dim_coords[dep_idx]
-                    elif old_dependency in old_aux_coords:
-                        dep_idx = old_aux_coords.index(old_dependency)
-                        new_dependency = new_aux_coords[dep_idx]
-                    else:
-                        dep_idx = scalar_coords.index(old_dependency)
-                        new_dependency = scalar_coords[dep_idx]
-                    new_dependencies[id(old_dependency)] = new_dependency
+        for _, _, factory in cube_signature.derived_coords_and_dims:
+            # Update the dependencies of the factory with coordinates of
+            # the concatenated cube. We need to check all coordinate types
+            # here (dim coords, aux coords, and scalar coords).
+
+            # Note: in contrast to other _build_... methods of this class, we
+            # do NOT need to distinguish between aux factories that span the
+            # nominated concatenation axis and aux factories that do not. The
+            # reason is that ALL aux factories need to be updated with the new
+            # coordinates of the concatenated cube (passed to this function via
+            # dim_coords_and_dims, aux_coords_and_dims, scalar_coords [these
+            # contain ALL new coordinates, not only the ones spanning the
+            # concatenation dimension]), so no special treatment for the aux
+            # factories that span the concatenation dimension is necessary. If
+            # not all aux factories are properly updated with references to the
+            # new coordinates, this may lead to KeyErrors (see
+            # https://github.com/SciTools/iris/issues/5339).
+            new_dependencies = {}
+            for old_dependency in factory.dependencies.values():
+                if old_dependency in old_dim_coords:
+                    dep_idx = old_dim_coords.index(old_dependency)
+                    new_dependency = new_dim_coords[dep_idx]
+                elif old_dependency in old_aux_coords:
+                    dep_idx = old_aux_coords.index(old_dependency)
+                    new_dependency = new_aux_coords[dep_idx]
+                else:
+                    dep_idx = old_scalar_coords.index(old_dependency)
+                    new_dependency = new_scalar_coords[dep_idx]
+                new_dependencies[id(old_dependency)] = new_dependency
 
-                # Create new factory with the updated dependencies.
-                factory = factory.updated(new_dependencies)
+            # Create new factory with the updated dependencies.
+            factory = factory.updated(new_dependencies)
 
             aux_factories.append(factory)
 
@@ -1307,7 +1353,7 @@ def _build_dim_coordinates(self):
 
         # Concatenate the points together for the nominated dimension.
         points = [
-            skeleton.signature.dim_coords[dim_ind].points
+            skeleton.signature.dim_coords[dim_ind].core_points()
             for skeleton in skeletons
         ]
         points = np.concatenate(tuple(points))
@@ -1316,7 +1362,7 @@ def _build_dim_coordinates(self):
         bounds = None
         if self._cube_signature.dim_coords[dim_ind].has_bounds():
             bounds = [
-                skeleton.signature.dim_coords[dim_ind].bounds
+                skeleton.signature.dim_coords[dim_ind].core_bounds()
                 for skeleton in skeletons
             ]
             bounds = np.concatenate(tuple(bounds))