Fix load_http bug, extend testing, and note to docs (#4580)

* Fix opendap bug, add docs and extra testing

* Add whats new entry

* Update docs/src/whatsnew/3.2.rst

Co-authored-by: Bill Little <[email protected]>

* Add warning box

Co-authored-by: Bill Little <[email protected]>

Author: lbdreyer

docs/src/whatsnew/3.2.rst

@@ -177,6 +177,9 @@ This document explains the changes made to Iris for this release
    to indicate cloud fraction greater than 7.9 oktas, rather than 7.5 
    (:issue:`3305`, :pull:`4535`)
 
+#. `@lbdreyer`_ fixed a bug in :class:`iris.io.load_http` which was missing an import
+   (:pull:`4580`)
+
 
 💣 Incompatible Changes
 =======================

lib/iris/__init__.py

@@ -44,6 +44,10 @@
         standard library function :func:`os.path.expanduser` and
         module :mod:`fnmatch` for more details.
 
+        .. warning::
+
+            If supplying a URL, only OPeNDAP Data Sources are supported.
+
     * constraints:
         Either a single constraint, or an iterable of constraints.
         Each constraint can be either a string, an instance of
@@ -287,6 +291,7 @@ def load(uris, constraints=None, callback=None):
 
     * uris:
         One or more filenames/URIs, as a string or :class:`pathlib.PurePath`.
+        If supplying a URL, only OPeNDAP Data Sources are supported.
 
     Kwargs:
 
@@ -315,6 +320,7 @@ def load_cube(uris, constraint=None, callback=None):
 
     * uris:
         One or more filenames/URIs, as a string or :class:`pathlib.PurePath`.
+        If supplying a URL, only OPeNDAP Data Sources are supported.
 
     Kwargs:
 
@@ -354,6 +360,7 @@ def load_cubes(uris, constraints=None, callback=None):
 
     * uris:
         One or more filenames/URIs, as a string or :class:`pathlib.PurePath`.
+        If supplying a URL, only OPeNDAP Data Sources are supported.
 
     Kwargs:
 
@@ -399,6 +406,7 @@ def load_raw(uris, constraints=None, callback=None):
 
     * uris:
         One or more filenames/URIs, as a string or :class:`pathlib.PurePath`.
+        If supplying a URL, only OPeNDAP Data Sources are supported.
 
     Kwargs:
 

lib/iris/fileformats/netcdf.py

@@ -825,12 +825,12 @@ def inner(cf_datavar):
 
 def load_cubes(filenames, callback=None, constraints=None):
     """
-    Loads cubes from a list of NetCDF filenames/URLs.
+    Loads cubes from a list of NetCDF filenames/OPeNDAP URLs.
 
     Args:
 
     * filenames (string/list):
-        One or more NetCDF filenames/DAP URLs to load from.
+        One or more NetCDF filenames/OPeNDAP URLs to load from.
 
     Kwargs:
 

lib/iris/io/__init__.py

@@ -216,7 +216,7 @@ def load_files(filenames, callback, constraints=None):
 
 def load_http(urls, callback):
     """
-    Takes a list of urls and a callback function, and returns a generator
+    Takes a list of OPeNDAP URLs and a callback function, and returns a generator
     of Cubes from the given URLs.
 
     .. note::
@@ -226,11 +226,11 @@ def load_http(urls, callback):
 
     """
     # Create default dict mapping iris format handler to its associated filenames
+    from iris.fileformats import FORMAT_AGENT
+
     handler_map = collections.defaultdict(list)
     for url in urls:
-        handling_format_spec = iris.fileformats.FORMAT_AGENT.get_spec(
-            url, None
-        )
+        handling_format_spec = FORMAT_AGENT.get_spec(url, None)
         handler_map[handling_format_spec].append(url)
 
     # Call each iris format handler with the appropriate filenames

lib/iris/tests/test_load.py

@@ -12,6 +12,9 @@
 import iris.tests as tests  # isort:skip
 
 import pathlib
+from unittest import mock
+
+import netCDF4
 
 import iris
 import iris.io
@@ -148,19 +151,20 @@ def test_path_object(self):
         self.assertEqual(len(cubes), 1)
 
 
-class TestOpenDAP(tests.IrisTest):
-    def test_load(self):
-        # Check that calling iris.load_* with a http URI triggers a call to
-        # ``iris.io.load_http``
+class TestOPeNDAP(tests.IrisTest):
+    def setUp(self):
+        self.url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15"
 
-        url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15"
+    def test_load_http_called(self):
+        # Check that calling iris.load_* with an http URI triggers a call to
+        # ``iris.io.load_http``
 
         class LoadHTTPCalled(Exception):
             pass
 
         def new_load_http(passed_urls, *args, **kwargs):
             self.assertEqual(len(passed_urls), 1)
-            self.assertEqual(url, passed_urls[0])
+            self.assertEqual(self.url, passed_urls[0])
             raise LoadHTTPCalled()
 
         try:
@@ -174,11 +178,28 @@ def new_load_http(passed_urls, *args, **kwargs):
                 iris.load_cubes,
             ]:
                 with self.assertRaises(LoadHTTPCalled):
-                    fn(url)
+                    fn(self.url)
 
         finally:
             iris.io.load_http = orig
 
+    def test_netCDF_Dataset_call(self):
+        # Check that load_http calls netCDF4.Dataset and supplies the expected URL.
+
+        # To avoid making a request to an OPeNDAP server in a test, instead
+        # mock the call to netCDF.Dataset so that it returns a dataset for a
+        # local file.
+        filename = tests.get_data_path(
+            ("NetCDF", "global", "xyt", "SMALL_total_column_co2.nc")
+        )
+        fake_dataset = netCDF4.Dataset(filename)
+
+        with mock.patch(
+            "netCDF4.Dataset", return_value=fake_dataset
+        ) as dataset_loader:
+            next(iris.io.load_http([self.url], callback=None))
+        dataset_loader.assert_called_with(self.url, mode="r")
+
 
 if __name__ == "__main__":
     tests.main()