Iris py38 (#3976)

* support for py38

* update CI and noxfile

* enforce alphabetical xml element attribute order

* full tests for py38 + fix docs-tests

* add whatsnew entry

* update doc-strings + review actions

* Alternate xml handling routine (#29)

* all xml tests pass for nox tests-3.8

* restored docstrings

* move sort_xml_attrs

* make sort_xml_attrs a classmethod

* update sort_xml_attr doc-string

Co-authored-by: Bill Little <[email protected]>

* add jamesp to whatsnew + minor tweak

Co-authored-by: James Penn <[email protected]>

Author: bjlittle

.cirrus.yml

@@ -98,6 +98,8 @@ linux_minimal_task:
       PY_VER: 3.6
     env:
       PY_VER: 3.7
+    env:
+      PY_VER: 3.8
   name: "${CIRRUS_OS}: py${PY_VER} tests (minimal)"
   container:
     image: gcc:latest
@@ -119,6 +121,8 @@ linux_task:
       PY_VER: 3.6
     env:
       PY_VER: 3.7
+    env:
+      PY_VER: 3.8
   name: "${CIRRUS_OS}: py${PY_VER} tests (full)"
   container:
     image: gcc:latest
@@ -146,9 +150,7 @@ linux_task:
 gallery_task:
   matrix:
     env:
-      PY_VER: 3.6
-    env:
-      PY_VER: 3.7
+      PY_VER: 3.8
   name: "${CIRRUS_OS}: py${PY_VER} doc tests (gallery)"
   container:
     image: gcc:latest
@@ -176,7 +178,7 @@ gallery_task:
 doctest_task:
   matrix:
     env:
-      PY_VER: 3.7
+      PY_VER: 3.8
   name: "${CIRRUS_OS}: py${PY_VER} doc tests"
   container:
     image: gcc:latest
@@ -210,7 +212,7 @@ doctest_task:
 link_task:
   matrix:
     env:
-      PY_VER: 3.7
+      PY_VER: 3.8
   name: "${CIRRUS_OS}: py${PY_VER} doc link check"
   container:
     image: gcc:latest

docs/src/common_links.inc

@@ -1,6 +1,7 @@
 .. comment
     Common resources in alphabetical order:
 
+.. _black: https://black.readthedocs.io/en/stable/
 .. _.cirrus.yml: https://github.com/SciTools/iris/blob/master/.cirrus.yml
 .. _.flake8.yml: https://github.com/SciTools/iris/blob/master/.flake8
 .. _cirrus-ci: https://cirrus-ci.com/github/SciTools/iris
@@ -19,6 +20,7 @@
 .. _legacy documentation: https://scitools.org.uk/iris/docs/v2.4.0/
 .. _matplotlib: https://matplotlib.org/
 .. _napolean: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/sphinxcontrib.napoleon.html
+.. _nox: https://nox.thea.codes/en/stable/
 .. _New Issue: https://github.com/scitools/iris/issues/new/choose
 .. _pull request: https://github.com/SciTools/iris/pulls
 .. _pull requests: https://github.com/SciTools/iris/pulls

docs/src/developers_guide/contributing_running_tests.rst

@@ -186,8 +186,6 @@ For further `nox`_ command-line options::
 .. note:: `nox`_ will cache its testing environments in the `.nox` root ``iris`` project directory.
 
 
-.. _black: https://black.readthedocs.io/en/stable/
-.. _nox: https://nox.thea.codes/en/latest/
 .. _setuptools: https://setuptools.readthedocs.io/en/latest/
 .. _tox: https://tox.readthedocs.io/en/latest/
 .. _virtualenv: https://virtualenv.pypa.io/en/latest/

docs/src/further_topics/metadata.rst

@@ -258,12 +258,12 @@ create a **new** instance directly from the metadata class itself,
     >>> DimCoordMetadata._make(values)
     DimCoordMetadata(standard_name=1, long_name=2, var_name=3, units=4, attributes=5, coord_system=6, climatological=7, circular=8)
 
-It is also possible to easily convert ``metadata`` to an `OrderedDict`_
+It is also possible to easily convert ``metadata`` to an `dict`_
 using the `namedtuple._asdict`_ method. This can be particularly handy when a
 standard Python built-in container is required to represent your ``metadata``,
 
     >>> metadata._asdict()
-    OrderedDict([('standard_name', 'longitude'), ('long_name', None), ('var_name', 'longitude'), ('units', Unit('degrees')), ('attributes', {'grinning face': '🙃'}), ('coord_system', GeogCS(6371229.0)), ('climatological', False), ('circular', False)])
+    {'standard_name': 'longitude', 'long_name': None, 'var_name': 'longitude', 'units': Unit('degrees'), 'attributes': {'grinning face': '🙃'}, 'coord_system': GeogCS(6371229.0), 'climatological': False, 'circular': False}
 
 Using the `namedtuple._replace`_ method allows you to create a new metadata
 class instance, but replacing specified members with **new** associated values,
@@ -943,7 +943,7 @@ such as a `dict`_,
 
     >>> mapping = latitude.metadata._asdict()
     >>> mapping
-    OrderedDict([('standard_name', 'latitude'), ('long_name', None), ('var_name', 'latitude'), ('units', Unit('degrees')), ('attributes', {}), ('coord_system', GeogCS(6371229.0)), ('climatological', False), ('circular', False)])
+    {'standard_name': 'latitude', 'long_name': None, 'var_name': 'latitude', 'units': Unit('degrees'), 'attributes': {}, 'coord_system': GeogCS(6371229.0), 'climatological': False, 'circular': False}
     >>> longitude.metadata = mapping
     >>> longitude.metadata
     DimCoordMetadata(standard_name='latitude', long_name=None, var_name='latitude', units=Unit('degrees'), attributes={}, coord_system=GeogCS(6371229.0), climatological=False, circular=False)
@@ -1000,7 +1000,6 @@ values. All other metadata members will be left unaltered.
 .. _NetCDF: https://www.unidata.ucar.edu/software/netcdf/
 .. _NetCDF CF Metadata Conventions: https://cfconventions.org/
 .. _NumPy: https://github.com/numpy/numpy
-.. _OrderedDict: https://docs.python.org/3/library/collections.html#collections.OrderedDict
 .. _Parametric Vertical Coordinate: https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#parametric-vertical-coordinate
 .. _rich comparison: https://www.python.org/dev/peps/pep-0207/
 .. _SciTools/iris: https://github.com/SciTools/iris

docs/src/installing.rst

@@ -16,8 +16,8 @@ any WSL_ distributions.
 
 .. _WSL: https://docs.microsoft.com/en-us/windows/wsl/install-win10
 
-.. note:: Iris currently supports and is tested against **Python 3.6** and
-          **Python 3.7**.  
+.. note:: Iris is currently supported and tested against Python ``3.6``,
+          ``3.7``, and ``3.8``.
 
 .. note:: This documentation was built using Python |python_version|.
 

docs/src/whatsnew/latest.rst

@@ -12,7 +12,6 @@ This document explains the changes made to Iris for this release
    :title: text-primary text-center font-weight-bold
    :body: bg-light
    :animate: fade-in
-   :open:
 
    The highlights for this major/minor release of Iris include:
 
@@ -96,7 +95,10 @@ This document explains the changes made to Iris for this release
 #. `@tkknight`_ moved the ``docs/iris`` directory to be in the parent
    directory ``docs``.  (:pull:`3975`)
 
-#. `@jamesp`_ updated a test to the latest numpy version (:pull:`3977`)
+#. `@jamesp`_ updated a test for `numpy`_ ``1.20.0``. (:pull:`3977`)
+
+#. `@bjlittle`_ and `@jamesp`_ extended the `cirrus-ci`_ testing and `nox`_
+   testing automation to support `Python 3.8`_. (:pull:`3976`)
 
 #. `@bjlittle`_ rationalised the ``noxfile.py``, and added the ability for
    each ``nox`` session to list its ``conda`` environment packages and
@@ -117,3 +119,5 @@ This document explains the changes made to Iris for this release
 .. _abstract base class: https://docs.python.org/3/library/abc.html
 .. _GitHub: https://github.com/SciTools/iris/issues/new/choose
 .. _Met Office: https://www.metoffice.gov.uk/
+.. _numpy: https://numpy.org/doc/stable/release/1.20.0-notes.html
+.. _Python 3.8: https://www.python.org/downloads/release/python-380/

lib/iris/coords.py

@@ -578,7 +578,21 @@ def shape(self):
         return self._values_dm.shape
 
     def xml_element(self, doc):
-        """Return a DOM element describing this metadata."""
+        """
+        Create the :class:`xml.dom.minidom.Element` that describes this
+        :class:`_DimensionalMetadata`.
+
+        Args:
+
+        * doc:
+            The parent :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Element` that will describe this
+            :class:`_DimensionalMetadata`, and the dictionary of attributes
+            that require to be added to this element.
+
+        """
         # Create the XML element as the camelCaseEquivalent of the
         # class name.
         element_name = type(self).__name__
@@ -881,6 +895,20 @@ def cube_dims(self, cube):
         return cube.cell_measure_dims(self)
 
     def xml_element(self, doc):
+        """
+        Create the :class:`xml.dom.minidom.Element` that describes this
+        :class:`CellMeasure`.
+
+        Args:
+
+        * doc:
+            The parent :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Element` that describes this
+            :class:`CellMeasure`.
+
+        """
         # Create the XML element as the camelCaseEquivalent of the
         # class name
         element = super().xml_element(doc=doc)
@@ -2228,14 +2256,26 @@ def nearest_neighbour_index(self, point):
         return result_index
 
     def xml_element(self, doc):
-        """Return a DOM element describing this Coord."""
+        """
+        Create the :class:`xml.dom.minidom.Element` that describes this
+        :class:`Coord`.
+
+        Args:
+
+        * doc:
+            The parent :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Element` that will describe this
+            :class:`DimCoord`, and the dictionary of attributes that require
+            to be added to this element.
+
+        """
         # Create the XML element as the camelCaseEquivalent of the
         # class name
         element = super().xml_element(doc=doc)
 
-        element.setAttribute("points", self._xml_array_repr(self.points))
-
-        # Add bounds handling
+        # Add bounds, points are handled by the parent class.
         if self.has_bounds():
             element.setAttribute("bounds", self._xml_array_repr(self.bounds))
 
@@ -2614,7 +2654,20 @@ def is_monotonic(self):
         return True
 
     def xml_element(self, doc):
-        """Return DOM element describing this :class:`iris.coords.DimCoord`."""
+        """
+        Create the :class:`xml.dom.minidom.Element` that describes this
+        :class:`DimCoord`.
+
+        Args:
+
+        * doc:
+            The parent :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Element` that describes this
+            :class:`DimCoord`.
+
+        """
         element = super().xml_element(doc)
         if self.circular:
             element.setAttribute("circular", str(self.circular))
@@ -2794,7 +2847,17 @@ def __add__(self, other):
 
     def xml_element(self, doc):
         """
-        Return a dom element describing itself
+        Create the :class:`xml.dom.minidom.Element` that describes this
+        :class:`CellMethod`.
+
+        Args:
+
+        * doc:
+            The parent :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Element` that describes this
+            :class:`CellMethod`.
 
         """
         cellMethod_xml_element = doc.createElement("cellMethod")

lib/iris/cube.py

@@ -225,6 +225,7 @@ def __getslice__(self, start, stop):
 
     def xml(self, checksum=False, order=True, byteorder=True):
         """Return a string of the XML that this list of cubes represents."""
+
         doc = Document()
         cubes_xml_element = doc.createElement("cubes")
         cubes_xml_element.setAttribute("xmlns", XML_NAMESPACE_URI)
@@ -239,6 +240,7 @@ def xml(self, checksum=False, order=True, byteorder=True):
         doc.appendChild(cubes_xml_element)
 
         # return our newly created XML string
+        doc = Cube._sort_xml_attrs(doc)
         return doc.toprettyxml(indent="  ")
 
     def extract(self, constraints):
@@ -755,6 +757,59 @@ class Cube(CFVariableMixin):
     #: is similar to Fortran or Matlab, but different than numpy.
     __orthogonal_indexing__ = True
 
+    @classmethod
+    def _sort_xml_attrs(cls, doc):
+        """
+        Takes an xml document and returns a copy with all element
+        attributes sorted in alphabetical order.
+
+        This is a private utility method required by iris to maintain
+        legacy xml behaviour beyond python 3.7.
+
+        Args:
+
+        * doc:
+            The :class:`xml.dom.minidom.Document`.
+
+        Returns:
+            The :class:`xml.dom.minidom.Document` with sorted element
+            attributes.
+
+        """
+        from xml.dom.minidom import Document
+
+        def _walk_nodes(node):
+            """Note: _walk_nodes is called recursively on child elements."""
+
+            # we don't want to copy the children here, so take a shallow copy
+            new_node = node.cloneNode(deep=False)
+
+            # Versions of python <3.8 order attributes in alphabetical order.
+            # Python >=3.8 order attributes in insert order.  For consistent behaviour
+            # across both, we'll go with alphabetical order always.
+            # Remove all the attribute nodes, then add back in alphabetical order.
+            attrs = [
+                new_node.getAttributeNode(attr_name).cloneNode(deep=True)
+                for attr_name in sorted(node.attributes.keys())
+            ]
+            for attr in attrs:
+                new_node.removeAttributeNode(attr)
+            for attr in attrs:
+                new_node.setAttributeNode(attr)
+
+            if node.childNodes:
+                children = [_walk_nodes(x) for x in node.childNodes]
+                for c in children:
+                    new_node.appendChild(c)
+
+            return new_node
+
+        nodes = _walk_nodes(doc.documentElement)
+        new_doc = Document()
+        new_doc.appendChild(nodes)
+
+        return new_doc
+
     def __init__(
         self,
         data,
@@ -3403,6 +3458,7 @@ def xml(self, checksum=False, order=True, byteorder=True):
         doc.appendChild(cube_xml_element)
 
         # Print our newly created XML
+        doc = self._sort_xml_attrs(doc)
         return doc.toprettyxml(indent="  ")
 
     def _xml_element(self, doc, checksum=False, order=True, byteorder=True):

lib/iris/tests/__init__.py

@@ -573,6 +573,10 @@ def assertXMLElement(self, obj, reference_filename):
         """
         doc = xml.dom.minidom.Document()
         doc.appendChild(obj.xml_element(doc))
+        # sort the attributes on xml elements before testing against known good state.
+        # this is to be compatible with stored test output where xml attrs are stored in alphabetical order,
+        # (which was default behaviour in python <3.8, but changed to insert order in >3.8)
+        doc = iris.cube.Cube._sort_xml_attrs(doc)
         pretty_xml = doc.toprettyxml(indent="  ")
         reference_path = self.get_result_path(reference_filename)
         self._check_same(

noxfile.py

@@ -19,7 +19,7 @@
 PACKAGE = str("lib" / Path("iris"))
 
 #: Cirrus-CI environment variable hook.
-PY_VER = os.environ.get("PY_VER", ["3.6", "3.7"])
+PY_VER = os.environ.get("PY_VER", ["3.6", "3.7", "3.8"])
 
 #: Default cartopy cache directory.
 CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy")