implement recursive search for GOES products across hour boundaries

src/metpy/remote/aws.py

@@ -493,11 +493,28 @@
         """
         return [item.rstrip(self.delimiter) for item in self.common_prefixes('')]
 
-    def _build_time_prefix(self, product, dt):
-        """Build the initial prefix for time and product."""
+    def _build_time_prefix(self, product, dt, depth=None):
+        """Build the initial prefix for time and product up to a particular depth.
+
+        Parameters
+        ----------
+        product : str
+            The product to search for
+        dt : datetime.datetime
+            The datetime to search for
+        depth : int, optional
+            The depth of the prefix to build. If None, builds the full prefix.
+            1: product
+            2: product/year
+            3: product/year/day_of_year
+            4: product/year/day_of_year/hour
+            5: product/year/day_of_year/hour/OR_product
+        """
         # Handle that the meso sector products are grouped in the same subdir
         reduced_product = product[:-1] if product.endswith(('M1', 'M2')) else product
         parts = [reduced_product, f'{dt:%Y}', f'{dt:%j}', f'{dt:%H}', f'OR_{product}']
+        if depth is not None:
+            return self.delimiter.join(parts[:depth])
         return self.delimiter.join(parts)
 
     def _subprod_prefix(self, prefix, mode, band):
@@ -557,9 +574,50 @@
 
         """
         dt = datetime.now(timezone.utc) if dt is None else ensure_timezone(dt)
-        time_prefix = self._build_time_prefix(product, dt)
-        prod_prefix = self._subprod_prefix(time_prefix, mode, band)
-        return self._closest_result(self.objects(prod_prefix), dt)
+
+        # We work with a list of keys/prefixes that we iteratively find that bound our target
+        # key. To start, this only contains the product.
+        bounding_keys = [self._build_time_prefix(product, dt, 1) + self.delimiter]
+
+        # Iteratively search with more specific keys, finding where our key fits within the
+        # list by using the common prefixes that exist for the current bounding keys
+        for depth in range(2, 5):  # Year, day of year, hour
+            # Get a key for the product/dt that we're looking for, constrained by how deep
+            # we are in the search i.e. product->year->day_of_year->hour->OR_product
+            search_key = self._build_time_prefix(product, dt, depth)
+
+            # Get the next collection of partial keys using the common prefixes for our
+            # candidates
+            prefixes = list(itertools.chain(*(self.common_prefixes(b) for b in bounding_keys)))
+
+            if not prefixes:  # No prefixes found, can't continue
+                raise ValueError(f'No data found for {product} at {dt}')
+
+            # Find where our target would be in the list and grab the ones on either side
+            # if possible. This also handles if we're off the end.
+            loc = bisect.bisect_left(prefixes, search_key)
+
+            # loc gives where our target *would* be in the list. Therefore slicing from loc - 1
+            # to loc + 1 gives the items to the left and right of our target. If we get 0,
+            # then there is nothing to the left and we only need the first item.
+            rng = slice(loc - 1, loc + 1) if loc else slice(0, 1)
+
+            # Make sure we don't go out of bounds
+            if loc >= len(prefixes):
+                rng = slice(len(prefixes) - 1, len(prefixes))
+
+            bounding_keys = prefixes[rng]
+
+        # Now that we have the bounding hour directories, we need to find the closest product
+        # Get all objects from the bounding keys with the appropriate mode and band
+        all_objects = []
+        for key in bounding_keys:
+            time_prefix = key.rstrip(self.delimiter)
+            prod_prefix = self._subprod_prefix(time_prefix, mode, band)
+            all_objects.extend(list(self.objects(prod_prefix)))
+
+        # Find the closest product to the requested time
+        return self._closest_result(all_objects, dt)
 
     def get_range(self, product, start, end, mode=None, band=None):
         """Yield products within a particular date/time range.

test_goes_client.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Test script to verify the robustness of the GOES client at hour boundaries.
+This script tests the recursive search implementation for finding products
+across hour boundaries.
+"""
+import sys
+from datetime import datetime, timezone, timedelta
+
+from metpy.remote import GOESArchive
+
+def test_goes_hour_boundary():
+    """Test the GOES client's ability to find products across hour boundaries."""
+    print("Testing GOES client at hour boundaries...")
+
+    # Create a GOES client
+    goes = GOESArchive(16)
+
+    # Test case 1: Exact hour boundary
+    # This would have failed with the old implementation if no products exist in the new hour
+    try:
+        # Use a time at exactly the top of an hour
+        dt = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+        print(f"\nTest 1: Searching at exact hour boundary: {dt}")
+        product = goes.get_product('ABI-L1b-RadC', dt, band=1)
+        print(f"Success! Found product: {product.name}")
+    except Exception as e:
+        print(f"Error: {e}")
+
+    # Test case 2: Just after hour boundary
+    try:
+        # Use a time just after the top of an hour
+        dt = datetime(2025, 1, 1, 0, 0, 30, tzinfo=timezone.utc)
+        print(f"\nTest 2: Searching just after hour boundary: {dt}")
+        product = goes.get_product('ABI-L1b-RadC', dt, band=1)
+        print(f"Success! Found product: {product.name}")
+    except Exception as e:
+        print(f"Error: {e}")
+
+    # Test case 3: Just before hour boundary
+    try:
+        # Use a time just before the top of an hour
+        dt = datetime(2025, 1, 1, 0, 59, 30, tzinfo=timezone.utc)
+        print(f"\nTest 3: Searching just before hour boundary: {dt}")
+        product = goes.get_product('ABI-L1b-RadC', dt, band=1)
+        print(f"Success! Found product: {product.name}")
+    except Exception as e:
+        print(f"Error: {e}")
+
+    # Test case 4: Day boundary
+    try:
+        # Use a time at day boundary
+        dt = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+        print(f"\nTest 4: Searching at day boundary: {dt}")
+        product = goes.get_product('ABI-L1b-RadC', dt, band=1)
+        print(f"Success! Found product: {product.name}")
+    except Exception as e:
+        print(f"Error: {e}")
+
+    print("\nAll tests completed!")
+
+if __name__ == "__main__":
+    test_goes_hour_boundary()