diff --git a/.gitignore b/.gitignore index ee821184de..7dcc87d981 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,7 @@ doc/_sidebar.rst.inc # Data files *.nc +!tests/sample_data/intake-esm/data/**/*.nc *.nc4 *.grb *.grib diff --git a/environment.yml b/environment.yml index 76a30d9f91..4b7324fd80 100644 --- a/environment.yml +++ b/environment.yml @@ -19,6 +19,7 @@ dependencies: - fire - geopy - humanfriendly + - intake >=2.0.0 - intake-esgf >=2025.10.22 - intake-esm - iris diff --git a/esmvalcore/io/intake_esm.py b/esmvalcore/io/intake_esm.py new file mode 100644 index 0000000000..83d78870a3 --- /dev/null +++ b/esmvalcore/io/intake_esm.py @@ -0,0 +1,264 @@ +"""Access data using `intake-esm `_. + +@CT: Needs updating, this is copied verbatim from intake-esgf module. + +.. note:: + + It is highly recommended that you take a moment to + :doc:`configure intake-esm ` before using it + with ESMValCore. Make sure to set ``local_cache`` to a path where + it can store downloaded files and if (some) Esm data is already + available on your system, point ``esg_dataroot`` to it. If you are + missing certain search results, you may want to choose a different + index node for searching the Esm. + +Run the command ``esmvaltool config copy data-intake-esm.yml`` to update +your :ref:`configuration ` to use this module. This will +create a file with the following content in your configuration directory: + +.. literalinclude:: ../configurations/data-intake-esm.yml + :language: yaml + :caption: Contents of ``data-intake-esm.yml`` + +""" + +from __future__ import annotations + +import copy +import warnings +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from esmvalcore.io.protocol import DataElement, DataSource +from esmvalcore.iris_helpers import dataset_to_iris + +if TYPE_CHECKING: + from pathlib import Path + + import iris.cube + from intake_esm.core import esm_datastore + from intake_esm.source import ESMDataSource + + from esmvalcore.typing import Facets, FacetValue + + +__all__ = [ + "IntakeEsmDataSource", + "IntakeEsmDataset", +] + + +def _to_path_dict( + esm_datastore: esm_datastore, + quiet: bool = False, +) -> dict[str, list[str | Path]]: + """Return the current search as a dictionary of paths to files. + + This method does not exist on intake-ESM's esm_datastore, so we implement it here. + """ + if not esm_datastore.keys() and not quiet: + warnings.warn( + "There are no datasets to load! Returning an empty dictionary.", + UserWarning, + stacklevel=2, + ) + return {} + + def _to_pathlist(source: ESMDataSource) -> list[str | Path]: + return source.df[source.path_column_name].to_list() + + return {key: _to_pathlist(val) for key, val in esm_datastore.items()} + + +@dataclass +class IntakeEsmDataset(DataElement): + """A dataset that can be used to load data found using intake-esm_. + + Roughly maps, conceptually, to `intake_esm.esm_datastore` + """ + + name: str + """A unique name identifying the data.""" + + facets: Facets = field(repr=False) + """Facets are key-value pairs that were used to find this data.""" + + catalog: esm_datastore = field(repr=False) + """The intake-esm catalog describing this data.""" + + _attributes: dict[str, Any] | None = field( + init=False, + repr=False, + default=None, + ) + + def __hash__(self) -> int: + """Return a number uniquely representing the data element.""" + return hash((self.name, self.facets.get("version"))) + + def prepare(self) -> None: + """Prepare the data for access. + + For intake-esm, no preparation is needed. + """ + + @property + def attributes(self) -> dict[str, Any]: + """Attributes are key-value pairs describing the data.""" + if self._attributes is None: + msg = ( + "Attributes have not been read yet. Call the `to_iris` method " + "first to read the attributes from the file." + ) + raise ValueError(msg) + return self._attributes + + @attributes.setter + def attributes(self, value: dict[str, Any]) -> None: + self._attributes = value + + def to_iris(self, quiet: bool = True) -> iris.cube.CubeList: + """Load the data as Iris cubes. + + Returns + ------- + : + The loaded data. + """ + files = _to_path_dict(self.catalog, quiet=quiet)[self.name] + + # Might want to pass through args/kwargs here? Ie. + dataset = self.catalog.to_dask() + # Store the local paths in the attributes for easier debugging. + dataset.attrs["source_file"] = ", ".join(str(f) for f in files) + # Cache the attributes. + self.attributes = copy.deepcopy(dataset.attrs) + return dataset_to_iris(dataset) + + +@dataclass +class IntakeEsmDataSource(DataSource): + """Data source that can be used to find data using intake-esm. + + Maps to an `intake_esm.esm_datasource`. + """ + + name: str + """A name identifying the data source.""" + + project: str + """The project that the data source provides data for.""" + + priority: int + """The priority of the data source. Lower values have priority.""" + + facets: dict[str, str] + """Mapping between the ESMValCore and intake-esm facet names.""" + + values: dict[str, dict[str, str]] = field(default_factory=dict) + """Mapping between the ESMValCore and intake-esm facet values.""" + + debug_info: str = field(init=False, repr=False, default="") + """A string containing debug information when no data is found.""" + + catalog: esm_datastore = field( + init=False, + repr=False, + ) + """The intake-esm catalog used to find data.""" + + def find_data(self, **facets: FacetValue) -> list[IntakeEsmDataset]: + """Find data. + + Parameters + ---------- + **facets : + Find data matching these facets. + + Returns + ------- + : + A list of data elements that have been found. + """ + # Select searchable facets and normalize so all values are `list[str]`. + normalized_facets = { + facet: [str(values)] + if isinstance(values, str | int | float) + else values + for facet, values in facets.items() + if facet in self.facets + } + + # Translate "our" facets to Esm facets and "our" values to Esm values. + query = { + their_facet: [ + self.values.get(our_facet, {}).get(v, v) + for v in normalized_facets[our_facet] + ] + for our_facet, their_facet in self.facets.items() + if our_facet in normalized_facets + } + + res = self.catalog.search(**query) + + if not len(res): + self.debug_info = ( + "`intake_esm.esm_datastore().search(" + + ", ".join( + [ + f"{k}={v}" if isinstance(v, list) else f"{k}='{v}'" + for k, v in query.items() + ], + ) + + ")` did not return any results." + ) + return [] + + # Return a list of datasets, with one IntakeEsmDataset per dataset_id. + result: list[IntakeEsmDataset] = [] + + # @CT: Made it to here, still some work to do after this bit + + inverse_values = { + our_facet: { + their_value: our_value + for our_value, their_value in self.values[our_facet].items() + } + for our_facet in self.values + } + + for key in sorted(res): + esm_datasource = res[key] + path_col = esm_datasource.path_column_name + + valid_paths = esm_datasource.df[path_col].to_list() + path_query = {path_col: valid_paths} + cat = self.catalog.search(**path_query) + + # Ensure only the requested variable is included in the dataset. + # https://github.com/esgf2-us/intake-esgf/blob/18437bff5ee75acaaceef63093101223b4692259/intake_esgf/catalog.py#L544-L552 + if "short_name" in normalized_facets: + cat = cat.search(**query) + + # Retrieve "our" facets associated with the dataset_id. + dataset_facets = {} + df = cat.df + for our_facet, esm_facet in self.facets.items(): + if esm_facet in df: + esm_values = df[esm_facet].unique().tolist() + our_values = [ + inverse_values.get(our_facet, {}).get(v, v) + for v in esm_values + ] + dataset_facets[our_facet] = our_values + + dataset = IntakeEsmDataset( + name=key, + facets={ + k: v[0] if len(v) == 1 else v + for k, v in dataset_facets.items() + }, # type: ignore[arg-type] + catalog=cat, + ) + result.append(dataset) + return result diff --git a/pyproject.toml b/pyproject.toml index 92b7e92757..0e26fe33d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,9 @@ dependencies = [ "fire", "geopy", "humanfriendly", + "intake>=2.0.0", "intake-esgf>=2025.10.22", - "intake-esm", + "intake-esm>=2025.2.3", "iris-grib>=0.20.0", # github.com/ESMValGroup/ESMValCore/issues/2535 "isodate>=0.7.0", "jinja2", diff --git a/tests/sample_data/intake-esm/catalog/cmip6-netcdf-test.csv b/tests/sample_data/intake-esm/catalog/cmip6-netcdf-test.csv new file mode 100644 index 0000000000..58729160d5 --- /dev/null +++ b/tests/sample_data/intake-esm/catalog/cmip6-netcdf-test.csv @@ -0,0 +1,60 @@ +activity_id,institution_id,source_id,experiment_id,member_id,table_id,variable_id,grid_label,path,time_range,dcpp_init_year,version +CMIP,BCC,BCC-ESM1,piControl,r1i1p1f1,Amon,tasmax,gn,./tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-ESM1/piControl/r1i1p1f1/Amon/tasmax/gn/v20181214/tasmax/tasmax_Amon_BCC-ESM1_piControl_r1i1p1f1_gn_185001-230012.nc,185001/230012,,v20181214 +CMIP,BCC,BCC-CSM2-MR,abrupt-4xCO2,r1i1p1f1,Amon,tasmax,gn,./tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-CSM2-MR/abrupt-4xCO2/r1i1p1f1/Amon/tasmax/gn/v20181016/tasmax/tasmax_Amon_BCC-CSM2-MR_abrupt-4xCO2_r1i1p1f1_gn_185001-200012.nc,185001/200012,,v20181016 +CMIP,CNRM-CERFACS,CNRM-CM6-1,historical,r4i1p1f2,Amon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/prsn/gr/v20190125/prsn/prsn_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc,185001/201412,,v20190125 +CMIP,CNRM-CERFACS,CNRM-CM6-1,historical,r4i1p1f2,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc,185001/201412,,v20190125 +CMIP,CNRM-CERFACS,CNRM-CM6-1,historical,r1i1p1f2,Lmon,gpp,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/gpp/gr/v20180917/gpp/gpp_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc,185001/201412,,v20180917 +CMIP,CNRM-CERFACS,CNRM-CM6-1,historical,r1i1p1f2,Lmon,residualFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/residualFrac/gr/v20180917/residualFrac/residualFrac_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc,185001/201412,,v20180917 +CMIP,CNRM-CERFACS,CNRM-CM6-1,historical,r6i1p1f2,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r6i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r6i1p1f2_gr_185001-201412.nc,185001/201412,,v20190125 +CMIP,CNRM-CERFACS,CNRM-CM6-1,piControl,r1i1p1f2,Lmon,landCoverFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/landCoverFrac/gr/v20180814/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc,185001/234912,,v20180814 +CMIP,CNRM-CERFACS,CNRM-CM6-1,piControl,r1i1p1f2,Lmon,gpp,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/gpp/gr/v20180814/gpp/gpp_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc,185001/234912,,v20180814 +CMIP,CNRM-CERFACS,CNRM-CM6-1,piControl,r1i1p1f2,Lmon,mrso,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/mrso/gr/v20180814/mrso/mrso_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc,185001/234912,,v20180814 +CMIP,CNRM-CERFACS,CNRM-CM6-1,1pctCO2,r1i1p1f2,Lmon,mrfso,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20180626/mrfso/mrfso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20180626 +CMIP,CNRM-CERFACS,CNRM-CM6-1,1pctCO2,r1i1p1f2,Lmon,landCoverFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20180626/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20180626 +CMIP,CNRM-CERFACS,CNRM-CM6-1,1pctCO2,r1i1p1f2,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20180626/evspsblveg/evspsblveg_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20180626 +CMIP,CNRM-CERFACS,CNRM-CM6-1,1pctCO2,r1i1p1f2,Lmon,gpp,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20180626/gpp/gpp_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20180626 +CMIP,CNRM-CERFACS,CNRM-CM6-1,1pctCO2,r1i1p1f2,Lmon,mrso,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrso/gr/v20180626/mrso/mrso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20180626 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,abrupt-4xCO2,r3i1p1f2,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/abrupt-4xCO2/r3i1p1f2/Amon/tasmax/gr/v20190208/tasmax/tasmax_Amon_CNRM-ESM2-1_abrupt-4xCO2_r3i1p1f2_gr_185001-199912.nc,185001/199912,,v20190208 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,historical,r3i1p1f2,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/historical/r3i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-ESM2-1_historical_r3i1p1f2_gr_185001-201412.nc,185001/201412,,v20190125 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,piControl,r1i1p1f2,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/evspsblveg/gr/v20181115/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc,185001/234912,,v20181115 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,piControl,r1i1p1f2,Lmon,residualFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/residualFrac/gr/v20181115/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc,185001/234912,,v20181115 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,amip,r1i1p1f2,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/amip/r1i1p1f2/Amon/tasmax/gr/v20181205/tasmax/tasmax_Amon_CNRM-ESM2-1_amip_r1i1p1f2_gr_197901-201412.nc,197901/201412,,v20181205 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r3i1p1f2,Omon,co3,gn,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r3i1p1f2/Omon/co3/gn/v20181107/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r3i1p1f2_gn_195001-199912.nc,195001/199912,,v20181107 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Omon,co3,gn,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Omon/co3/gn/v20181018/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gn_195001-199912.nc,195001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Lmon,mrfso,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20181018/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Lmon,landCoverFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20181018/landCoverFrac/landCoverFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20181018/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Lmon,gpp,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20181018/gpp/gpp_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r1i1p1f2,Lmon,residualFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/residualFrac/gr/v20181018/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc,185001/199912,,v20181018 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r2i1p1f2,Omon,co3,gn,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Omon/co3/gn/v20181031/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gn_195001-199912.nc,195001/199912,,v20181031 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r2i1p1f2,Lmon,mrfso,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/mrfso/gr/v20181031/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc,185001/199912,,v20181031 +CMIP,CNRM-CERFACS,CNRM-ESM2-1,1pctCO2,r2i1p1f2,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/evspsblveg/gr/v20181031/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc,185001/199912,,v20181031 +CMIP,IPSL,IPSL-CM6A-LR,abrupt-4xCO2,r2i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r2i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r2i1p1f1_gr_185002-185501.nc,185002/185501,,v20180914 +CMIP,IPSL,IPSL-CM6A-LR,abrupt-4xCO2,r9i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r9i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r9i1p1f1_gr_185009-185508.nc,185009/185508,,v20180914 +CMIP,IPSL,IPSL-CM6A-LR,historical,r15i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r15i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r15i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r3i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r3i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r3i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r21i1p1f1,Omon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r21i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r21i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r23i1p1f1,Omon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r23i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r23i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r24i1p1f1,Amon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r24i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r24i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r22i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r22i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r22i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r31i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r31i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r31i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r9i1p1f1,Amon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r9i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r9i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r7i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r7i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r7i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r18i1p1f1,Amon,tasmax,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r18i1p1f1/Amon/tasmax/gr/v20180803/tasmax/tasmax_Amon_IPSL-CM6A-LR_historical_r18i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,historical,r6i1p1f1,Omon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r6i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r6i1p1f1_gr_185001-201412.nc,185001/201412,,v20180803 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Omon,co3,gn,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/co3/gn/v20180802/co3/co3_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gn_225001-234912.nc,225001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Omon,prra,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/prra/gr/v20181123/prra/prra_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_285001-304912.nc,285001/304912,,v20181123 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Lmon,landCoverFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/landCoverFrac/gr/v20180802/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc,185001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/evspsblveg/gr/v20180802/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc,185001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Lmon,gpp,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/gpp/gr/v20180802/gpp/gpp_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc,185001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Lmon,residualFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/residualFrac/gr/v20180802/residualFrac/residualFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc,185001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,piControl,r1i1p1f1,Lmon,mrso,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/mrso/gr/v20180802/mrso/mrso_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc,185001/234912,,v20180802 +CMIP,IPSL,IPSL-CM6A-LR,amip,r10i1p1f1,Amon,prsn,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/amip/r10i1p1f1/Amon/prsn/gr/v20181109/prsn/prsn_Amon_IPSL-CM6A-LR_amip_r10i1p1f1_gr_195801-201412.nc,195801/201412,,v20181109 +CMIP,IPSL,IPSL-CM6A-LR,1pctCO2,r1i1p1f1,Omon,co3,gn,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Omon/co3/gn/v20180727/co3/co3_Omon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gn_195001-199912.nc,195001/199912,,v20180727 +CMIP,IPSL,IPSL-CM6A-LR,1pctCO2,r1i1p1f1,Lmon,landCoverFrac,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/landCoverFrac/gr/v20180727/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc,185001/199912,,v20180727 +CMIP,IPSL,IPSL-CM6A-LR,1pctCO2,r1i1p1f1,Lmon,evspsblveg,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/evspsblveg/gr/v20180727/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc,185001/199912,,v20180727 +CMIP,IPSL,IPSL-CM6A-LR,1pctCO2,r1i1p1f1,Lmon,mrso,gr,./tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/mrso/gr/v20180727/mrso/mrso_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc,185001/199912,,v20180727 +CMIP,MIROC,MIROC6,historical,r4i1p1f1,Amon,prsn,gn,./tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/historical/r4i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_historical_r4i1p1f1_gn_195001-201412.nc,195001/201412,,v20190311 +CMIP,MIROC,MIROC6,amip,r7i1p1f1,Amon,prsn,gn,./tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/amip/r7i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_amip_r7i1p1f1_gn_197901-201412.nc,197901/201412,,v20190311 +CMIP,MRI,MRI-ESM2-0,historical,r1i1p1f1,Amon,tasmax,gn,./tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r1i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r1i1p1f1_gn_185001-201412.nc,185001/201412,,v20190222 +CMIP,MRI,MRI-ESM2-0,historical,r3i1p1f1,Amon,tasmax,gn,./tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r3i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r3i1p1f1_gn_185001-201412.nc,185001/201412,,v20190222 diff --git a/tests/sample_data/intake-esm/catalog/cmip6-netcdf.json b/tests/sample_data/intake-esm/catalog/cmip6-netcdf.json new file mode 100644 index 0000000000..4ae05b5785 --- /dev/null +++ b/tests/sample_data/intake-esm/catalog/cmip6-netcdf.json @@ -0,0 +1,80 @@ +{ + "esmcat_version": "0.1.0", + "id": "sample-cmip6", + "description": "This is a sample ESM catalog for CMIP6 data in netcdf format", + "catalog_file": "cmip6-netcdf-test.csv", + "attributes": [ + { + "column_name": "activity_id", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_activity_id.json" + }, + { + "column_name": "source_id", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_source_id.json" + }, + { + "column_name": "institution_id", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_institution_id.json" + }, + { + "column_name": "experiment_id", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_experiment_id.json" + }, + { + "column_name": "member_id", + "vocabulary": "" + }, + { + "column_name": "table_id", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_table_id.json" + }, + { + "column_name": "variable_id", + "vocabulary": "" + }, + { + "column_name": "grid_label", + "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_grid_label.json" + }, + { + "column_name": "version", + "vocabulary": "" + } + ], + "assets": { + "column_name": "path", + "format": "netcdf" + }, + "aggregation_control": { + "variable_column_name": "variable_id", + "groupby_attrs": [ + "activity_id", + "institution_id", + "source_id", + "experiment_id", + "table_id", + "grid_label" + ], + "aggregations": [ + { + "type": "join_new", + "attribute_name": "member_id", + "options": { + "coords": "minimal", + "compat": "override" + } + }, + { + "type": "join_existing", + "attribute_name": "time_range", + "options": { + "dim": "time" + } + }, + { + "type": "union", + "attribute_name": "variable_id" + } + ] + } +} diff --git a/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-CSM2-MR/abrupt-4xCO2/r1i1p1f1/Amon/tasmax/gn/v20181016/tasmax/tasmax_Amon_BCC-CSM2-MR_abrupt-4xCO2_r1i1p1f1_gn_185001-200012.nc b/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-CSM2-MR/abrupt-4xCO2/r1i1p1f1/Amon/tasmax/gn/v20181016/tasmax/tasmax_Amon_BCC-CSM2-MR_abrupt-4xCO2_r1i1p1f1_gn_185001-200012.nc new file mode 100644 index 0000000000..15f313ffdd Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-CSM2-MR/abrupt-4xCO2/r1i1p1f1/Amon/tasmax/gn/v20181016/tasmax/tasmax_Amon_BCC-CSM2-MR_abrupt-4xCO2_r1i1p1f1_gn_185001-200012.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-ESM1/piControl/r1i1p1f1/Amon/tasmax/gn/v20181214/tasmax/tasmax_Amon_BCC-ESM1_piControl_r1i1p1f1_gn_185001-230012.nc b/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-ESM1/piControl/r1i1p1f1/Amon/tasmax/gn/v20181214/tasmax/tasmax_Amon_BCC-ESM1_piControl_r1i1p1f1_gn_185001-230012.nc new file mode 100644 index 0000000000..337ccf14f5 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/BCC/BCC-ESM1/piControl/r1i1p1f1/Amon/tasmax/gn/v20181214/tasmax/tasmax_Amon_BCC-ESM1_piControl_r1i1p1f1_gn_185001-230012.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20180626/evspsblveg/evspsblveg_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20180626/evspsblveg/evspsblveg_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..a0903bd9b6 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20180626/evspsblveg/evspsblveg_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20180626/gpp/gpp_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20180626/gpp/gpp_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..5d9f500e93 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20180626/gpp/gpp_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20180626/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20180626/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..5e75144caf Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20180626/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20180626/mrfso/mrfso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20180626/mrfso/mrfso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..b8296d28f4 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20180626/mrfso/mrfso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrso/gr/v20180626/mrso/mrso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrso/gr/v20180626/mrso/mrso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..1144df2514 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/1pctCO2/r1i1p1f2/Lmon/mrso/gr/v20180626/mrso/mrso_Lmon_CNRM-CM6-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/gpp/gr/v20180917/gpp/gpp_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/gpp/gr/v20180917/gpp/gpp_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..e93b3c2cc0 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/gpp/gr/v20180917/gpp/gpp_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/residualFrac/gr/v20180917/residualFrac/residualFrac_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/residualFrac/gr/v20180917/residualFrac/residualFrac_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..3e400137cb Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r1i1p1f2/Lmon/residualFrac/gr/v20180917/residualFrac/residualFrac_Lmon_CNRM-CM6-1_historical_r1i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/prsn/gr/v20190125/prsn/prsn_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/prsn/gr/v20190125/prsn/prsn_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..a6a51e1cd8 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/prsn/gr/v20190125/prsn/prsn_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..91186e7a4c Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r4i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r4i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r6i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r6i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r6i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r6i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..01a2859ba7 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/historical/r6i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-CM6-1_historical_r6i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/gpp/gr/v20180814/gpp/gpp_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/gpp/gr/v20180814/gpp/gpp_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc new file mode 100644 index 0000000000..4679edf558 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/gpp/gr/v20180814/gpp/gpp_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/landCoverFrac/gr/v20180814/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/landCoverFrac/gr/v20180814/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc new file mode 100644 index 0000000000..5ca1910cd1 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/landCoverFrac/gr/v20180814/landCoverFrac/landCoverFrac_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/mrso/gr/v20180814/mrso/mrso_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/mrso/gr/v20180814/mrso/mrso_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc new file mode 100644 index 0000000000..4df7d3d41a Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-CM6-1/piControl/r1i1p1f2/Lmon/mrso/gr/v20180814/mrso/mrso_Lmon_CNRM-CM6-1_piControl_r1i1p1f2_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20181018/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20181018/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..9ca2d82dbc Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/evspsblveg/gr/v20181018/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20181018/gpp/gpp_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20181018/gpp/gpp_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..f6322ae017 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/gpp/gr/v20181018/gpp/gpp_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20181018/landCoverFrac/landCoverFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20181018/landCoverFrac/landCoverFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..ed204d92dc Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/landCoverFrac/gr/v20181018/landCoverFrac/landCoverFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20181018/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20181018/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..b1bc53b5dd Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/mrfso/gr/v20181018/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/residualFrac/gr/v20181018/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/residualFrac/gr/v20181018/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..0447ac35e3 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Lmon/residualFrac/gr/v20181018/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Omon/co3/gn/v20181018/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gn_195001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Omon/co3/gn/v20181018/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gn_195001-199912.nc new file mode 100644 index 0000000000..bf59de4135 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r1i1p1f2/Omon/co3/gn/v20181018/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r1i1p1f2_gn_195001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/evspsblveg/gr/v20181031/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/evspsblveg/gr/v20181031/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..72d455b1f9 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/evspsblveg/gr/v20181031/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/mrfso/gr/v20181031/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/mrfso/gr/v20181031/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..6ecc349a29 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Lmon/mrfso/gr/v20181031/mrfso/mrfso_Lmon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Omon/co3/gn/v20181031/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gn_195001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Omon/co3/gn/v20181031/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gn_195001-199912.nc new file mode 100644 index 0000000000..b9526feb3e Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r2i1p1f2/Omon/co3/gn/v20181031/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r2i1p1f2_gn_195001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r3i1p1f2/Omon/co3/gn/v20181107/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r3i1p1f2_gn_195001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r3i1p1f2/Omon/co3/gn/v20181107/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r3i1p1f2_gn_195001-199912.nc new file mode 100644 index 0000000000..3680667062 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/1pctCO2/r3i1p1f2/Omon/co3/gn/v20181107/co3/co3_Omon_CNRM-ESM2-1_1pctCO2_r3i1p1f2_gn_195001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/abrupt-4xCO2/r3i1p1f2/Amon/tasmax/gr/v20190208/tasmax/tasmax_Amon_CNRM-ESM2-1_abrupt-4xCO2_r3i1p1f2_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/abrupt-4xCO2/r3i1p1f2/Amon/tasmax/gr/v20190208/tasmax/tasmax_Amon_CNRM-ESM2-1_abrupt-4xCO2_r3i1p1f2_gr_185001-199912.nc new file mode 100644 index 0000000000..bcaff5169a Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/abrupt-4xCO2/r3i1p1f2/Amon/tasmax/gr/v20190208/tasmax/tasmax_Amon_CNRM-ESM2-1_abrupt-4xCO2_r3i1p1f2_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/amip/r1i1p1f2/Amon/tasmax/gr/v20181205/tasmax/tasmax_Amon_CNRM-ESM2-1_amip_r1i1p1f2_gr_197901-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/amip/r1i1p1f2/Amon/tasmax/gr/v20181205/tasmax/tasmax_Amon_CNRM-ESM2-1_amip_r1i1p1f2_gr_197901-201412.nc new file mode 100644 index 0000000000..d1976e6663 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/amip/r1i1p1f2/Amon/tasmax/gr/v20181205/tasmax/tasmax_Amon_CNRM-ESM2-1_amip_r1i1p1f2_gr_197901-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/historical/r3i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-ESM2-1_historical_r3i1p1f2_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/historical/r3i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-ESM2-1_historical_r3i1p1f2_gr_185001-201412.nc new file mode 100644 index 0000000000..de0eb59120 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/historical/r3i1p1f2/Amon/tasmax/gr/v20190125/tasmax/tasmax_Amon_CNRM-ESM2-1_historical_r3i1p1f2_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/evspsblveg/gr/v20181115/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/evspsblveg/gr/v20181115/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc new file mode 100644 index 0000000000..14452e63d8 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/evspsblveg/gr/v20181115/evspsblveg/evspsblveg_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/residualFrac/gr/v20181115/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/residualFrac/gr/v20181115/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc new file mode 100644 index 0000000000..db8d68ef70 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/CNRM-CERFACS/CNRM-ESM2-1/piControl/r1i1p1f2/Lmon/residualFrac/gr/v20181115/residualFrac/residualFrac_Lmon_CNRM-ESM2-1_piControl_r1i1p1f2_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/evspsblveg/gr/v20180727/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/evspsblveg/gr/v20180727/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc new file mode 100644 index 0000000000..4ba33a45f1 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/evspsblveg/gr/v20180727/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/landCoverFrac/gr/v20180727/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/landCoverFrac/gr/v20180727/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc new file mode 100644 index 0000000000..51aabe6a51 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/landCoverFrac/gr/v20180727/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/mrso/gr/v20180727/mrso/mrso_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/mrso/gr/v20180727/mrso/mrso_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc new file mode 100644 index 0000000000..1304f452a2 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Lmon/mrso/gr/v20180727/mrso/mrso_Lmon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gr_185001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Omon/co3/gn/v20180727/co3/co3_Omon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gn_195001-199912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Omon/co3/gn/v20180727/co3/co3_Omon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gn_195001-199912.nc new file mode 100644 index 0000000000..472db80151 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/1pctCO2/r1i1p1f1/Omon/co3/gn/v20180727/co3/co3_Omon_IPSL-CM6A-LR_1pctCO2_r1i1p1f1_gn_195001-199912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r2i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r2i1p1f1_gr_185002-185501.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r2i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r2i1p1f1_gr_185002-185501.nc new file mode 100644 index 0000000000..932161063d Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r2i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r2i1p1f1_gr_185002-185501.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r9i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r9i1p1f1_gr_185009-185508.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r9i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r9i1p1f1_gr_185009-185508.nc new file mode 100644 index 0000000000..4754ae0ffa Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/abrupt-4xCO2/r9i1p1f1/Omon/prra/gr/v20180914/prra/prra_Omon_IPSL-CM6A-LR_abrupt-4xCO2_r9i1p1f1_gr_185009-185508.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/amip/r10i1p1f1/Amon/prsn/gr/v20181109/prsn/prsn_Amon_IPSL-CM6A-LR_amip_r10i1p1f1_gr_195801-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/amip/r10i1p1f1/Amon/prsn/gr/v20181109/prsn/prsn_Amon_IPSL-CM6A-LR_amip_r10i1p1f1_gr_195801-201412.nc new file mode 100644 index 0000000000..7d0117f47a Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/amip/r10i1p1f1/Amon/prsn/gr/v20181109/prsn/prsn_Amon_IPSL-CM6A-LR_amip_r10i1p1f1_gr_195801-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r15i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r15i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r15i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r15i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..3a7165a829 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r15i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r15i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r18i1p1f1/Amon/tasmax/gr/v20180803/tasmax/tasmax_Amon_IPSL-CM6A-LR_historical_r18i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r18i1p1f1/Amon/tasmax/gr/v20180803/tasmax/tasmax_Amon_IPSL-CM6A-LR_historical_r18i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..426be59e13 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r18i1p1f1/Amon/tasmax/gr/v20180803/tasmax/tasmax_Amon_IPSL-CM6A-LR_historical_r18i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r21i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r21i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r21i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r21i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..0f0c71476c Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r21i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r21i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r22i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r22i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r22i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r22i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..488cddce11 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r22i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r22i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r23i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r23i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r23i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r23i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..138a221285 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r23i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r23i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r24i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r24i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r24i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r24i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..4f5d051838 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r24i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r24i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r31i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r31i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r31i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r31i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..d4c87fb212 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r31i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r31i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r3i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r3i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r3i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r3i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..32a299ab2f Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r3i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r3i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r6i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r6i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r6i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r6i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..b9dac8047f Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r6i1p1f1/Omon/prsn/gr/v20180803/prsn/prsn_Omon_IPSL-CM6A-LR_historical_r6i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r7i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r7i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r7i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r7i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..0bdba70548 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r7i1p1f1/Omon/prra/gr/v20180803/prra/prra_Omon_IPSL-CM6A-LR_historical_r7i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r9i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r9i1p1f1_gr_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r9i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r9i1p1f1_gr_185001-201412.nc new file mode 100644 index 0000000000..748ba9821b Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/historical/r9i1p1f1/Amon/prsn/gr/v20180803/prsn/prsn_Amon_IPSL-CM6A-LR_historical_r9i1p1f1_gr_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/evspsblveg/gr/v20180802/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/evspsblveg/gr/v20180802/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc new file mode 100644 index 0000000000..b8c8f1d5ce Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/evspsblveg/gr/v20180802/evspsblveg/evspsblveg_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/gpp/gr/v20180802/gpp/gpp_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/gpp/gr/v20180802/gpp/gpp_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc new file mode 100644 index 0000000000..a150275d13 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/gpp/gr/v20180802/gpp/gpp_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/landCoverFrac/gr/v20180802/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/landCoverFrac/gr/v20180802/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc new file mode 100644 index 0000000000..64e8cbc6c4 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/landCoverFrac/gr/v20180802/landCoverFrac/landCoverFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/mrso/gr/v20180802/mrso/mrso_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/mrso/gr/v20180802/mrso/mrso_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc new file mode 100644 index 0000000000..694242ae52 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/mrso/gr/v20180802/mrso/mrso_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/residualFrac/gr/v20180802/residualFrac/residualFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/residualFrac/gr/v20180802/residualFrac/residualFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc new file mode 100644 index 0000000000..768ced2109 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Lmon/residualFrac/gr/v20180802/residualFrac/residualFrac_Lmon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_185001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/co3/gn/v20180802/co3/co3_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gn_225001-234912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/co3/gn/v20180802/co3/co3_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gn_225001-234912.nc new file mode 100644 index 0000000000..9d38da04c3 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/co3/gn/v20180802/co3/co3_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gn_225001-234912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/prra/gr/v20181123/prra/prra_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_285001-304912.nc b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/prra/gr/v20181123/prra/prra_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_285001-304912.nc new file mode 100644 index 0000000000..b9f2e1cf0f Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/IPSL/IPSL-CM6A-LR/piControl/r1i1p1f1/Omon/prra/gr/v20181123/prra/prra_Omon_IPSL-CM6A-LR_piControl_r1i1p1f1_gr_285001-304912.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/amip/r7i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_amip_r7i1p1f1_gn_197901-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/amip/r7i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_amip_r7i1p1f1_gn_197901-201412.nc new file mode 100644 index 0000000000..2e3b9d061c Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/amip/r7i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_amip_r7i1p1f1_gn_197901-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/historical/r4i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_historical_r4i1p1f1_gn_195001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/historical/r4i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_historical_r4i1p1f1_gn_195001-201412.nc new file mode 100644 index 0000000000..e2b1e14fc7 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/MIROC/MIROC6/historical/r4i1p1f1/Amon/prsn/gn/v20190311/prsn/prsn_Amon_MIROC6_historical_r4i1p1f1_gn_195001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r1i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r1i1p1f1_gn_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r1i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r1i1p1f1_gn_185001-201412.nc new file mode 100644 index 0000000000..2175121666 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r1i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r1i1p1f1_gn_185001-201412.nc differ diff --git a/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r3i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r3i1p1f1_gn_185001-201412.nc b/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r3i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r3i1p1f1_gn_185001-201412.nc new file mode 100644 index 0000000000..611866ecd6 Binary files /dev/null and b/tests/sample_data/intake-esm/data/CMIP6/MRI/MRI-ESM2-0/historical/r3i1p1f1/Amon/tasmax/gn/v20190222/tasmax/tasmax_Amon_MRI-ESM2-0_historical_r3i1p1f1_gn_185001-201412.nc differ diff --git a/tests/unit/io/test_intake_esm.py b/tests/unit/io/test_intake_esm.py new file mode 100644 index 0000000000..31df36c1ff --- /dev/null +++ b/tests/unit/io/test_intake_esm.py @@ -0,0 +1,207 @@ +"""Unit tests for esmvalcore.io.intake_esm.""" + +from __future__ import annotations + +import importlib.resources +from pathlib import Path +from typing import TYPE_CHECKING + +import intake +import pytest +import xarray as xr + +import esmvalcore.io.intake_esm +from esmvalcore.io.intake_esm import ( + IntakeEsmDataset, + IntakeEsmDataSource, + _to_path_dict, +) + +if TYPE_CHECKING: + from intake_esm.core import esm_datastore + from pytest_mock import MockerFixture + + +with importlib.resources.as_file( + importlib.resources.files("tests"), +) as test_dir: + esm_ds_fhandle = ( + Path(test_dir) + / "sample_data" + / "intake-esm" + / "catalog" + / "cmip6-netcdf.json" + ) + + +def test_intake_esm_dataset_repr() -> None: + cat = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + dataset = IntakeEsmDataset(name="id", facets={}, catalog=cat) + assert repr(dataset) == "IntakeEsmDataset(name='id')" + + +def test_prepare(mocker: MockerFixture) -> None: + """IntakeEsmDataset.prepare should not do anything (just pass).""" + cat = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + dataset = IntakeEsmDataset(name="id", facets={}, catalog=cat) + + # prepare() just passes for intake-esm, so we just verify it doesn't raise + dataset.prepare() + + +def test_attributes_raises_before_to_iris() -> None: + """Accessing attributes before to_iris should raise ValueError.""" + cat = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + dataset = IntakeEsmDataset(name="id", facets={}, catalog=cat) + with pytest.raises(ValueError, match="Attributes have not been read yet"): + _ = dataset.attributes + + +def test_to_iris(mocker: MockerFixture) -> None: + """`to_iris` should load the data and cache attributes.""" + cat = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + key = "my.dataset.1" + mocker.patch( + "esmvalcore.io.intake_esm._to_path_dict", + return_value={key: ["/path/to/file.nc"]}, + ) + ds = xr.Dataset(attrs={"attr": "value"}) + mocker.patch.object(cat, "to_dask", return_value=ds) + + cubes = mocker.sentinel.cubes + mocker.patch.object( + esmvalcore.io.intake_esm, + "dataset_to_iris", + return_value=cubes, + ) + + dataset = IntakeEsmDataset(name=key, facets={}, catalog=cat) + result = dataset.to_iris() + assert result is cubes + + assert dataset.attributes == { + "attr": "value", + "source_file": "/path/to/file.nc", + } + + +def test_find_data_no_results_sets_debug_info() -> None: + """When catalog.search returns empty results, find_data should return empty list and set debug_info.""" + data_source = IntakeEsmDataSource( + name="src", + project="CMIP6", + priority=1, + facets={"short_name": "variable_id"}, + ) + + cat: esm_datastore = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + + data_source.catalog = cat + + result = data_source.find_data(short_name="non_existent_variable") + assert result == [] + expected_debug_info = "`intake_esm.esm_datastore().search(variable_id=['non_existent_variable'])` did not return any results." + assert data_source.debug_info == expected_debug_info + + +def test_find_data() -> None: + """find_data should convert catalog.df rows into IntakeEsmDataset instances. + + CT Note: I'm not sure what project should be in here? + """ + cat: esm_datastore = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + + data_source = IntakeEsmDataSource( + name="src", + project="CMIP6", + priority=1, + facets={ + "activity": "activity_id", + "dataset": "source_id", + "ensemble": "member_id", + "exp": "experiment_id", + "institute": "institution_id", + "grid": "grid_label", + "mip": "table_id", + "short_name": "variable_id", + "version": "version", + }, + values={}, + ) + data_source.catalog = cat + + # Call find_data - it should use the df we set and return 8 datasets + results = data_source.find_data(short_name="tasmax") + assert isinstance(results, list) + assert len(results) == 8 + + dataset = results[0] + assert isinstance(dataset, IntakeEsmDataset) + assert dataset.name == "CMIP.BCC.BCC-CSM2-MR.abrupt-4xCO2.Amon.gn" + + assert hash(dataset) == hash((dataset.name, "v20181016")) + + assert dataset.facets == { + "activity": "CMIP", + "dataset": "BCC-CSM2-MR", + "ensemble": "r1i1p1f1", + "exp": "abrupt-4xCO2", + "grid": "gn", + "institute": "BCC", + "mip": "Amon", + "short_name": "tasmax", + "version": "v20181016", + } + + +def test_to_iris_nomock(): + """`to_iris` should load data from a real intake-esm catalog.""" + cat: esm_datastore = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + + data_source = IntakeEsmDataSource( + name="src", + project="CMIP6", + priority=1, + facets={ + "activity": "activity_id", + "dataset": "source_id", + "ensemble": "member_id", + "exp": "experiment_id", + "institute": "institution_id", + "grid": "grid_label", + "mip": "table_id", + "short_name": "variable_id", + "timerange": "time_range", + "version": "version", + }, + values={}, + ) + data_source.catalog = cat + + # Call find_data - it should use the df we set and return 8 datasets. + # Then we'll load the first one. + results = data_source.find_data(short_name="tasmax") + dataset = results[0] + assert isinstance(dataset, IntakeEsmDataset) + + # Raises a KeyError because the dtype of the dataset is Object, which I don't think NCData likes. + with pytest.raises(KeyError, match="'O'"): + dataset.to_iris() + + +def test_to_path_dict_nofiles() -> None: + """Test for quiet flag. + + If we disable the `quiet` flag and pass a search query that returns no results, `to_path_dict` + should warn. + + TODO: Can this code path ever be triggered in practice? + """ + cat: esm_datastore = intake.open_esm_datastore(esm_ds_fhandle.as_posix()) + + empty_cat = cat.search(variable_id="non_existent_variable") + + with pytest.warns(UserWarning, match="There are no datasets to load!"): + ret = _to_path_dict(empty_cat, quiet=False) + + assert ret == {}