From b18ce5c5d9776d58ac923d075bae4d1bf2250a03 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 10:35:19 +0800 Subject: [PATCH 01/10] v3.9.0 --- CHANGELOG.md | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70534e90..17d46a10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ## [Unreleased] + +## [v3.9.0] - 2025-06-13 + ### Added - Added separate `recommendation` field in error messages when running in non-verbose mode [#257](https://github.com/stac-utils/stac-validator/pull/257) - Verbose error messages for JSONSchemaValidationErrors [#257](https://github.com/stac-utils/stac-validator/pull/257) @@ -263,7 +266,8 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) - With the newest version - 1.0.0-beta.2 - items will run through jsonchema validation before the PySTAC validation. The reason for this is that jsonschema will give more informative error messages. This should be addressed better in the future. This is not the case with the --recursive option as time can be a concern here with larger collections. - Logging. Various additions were made here depending on the options selected. This was done to help assist people to update their STAC collections. -[Unreleased]: https://github.com/sparkgeo/stac-validator/compare/v3.8.1..main +[Unreleased]: https://github.com/sparkgeo/stac-validator/compare/v3.9.0..main +[v3.9.0]: https://github.com/sparkgeo/stac-validator/compare/v3.8.1..v3.9.0 [v3.8.1]: https://github.com/sparkgeo/stac-validator/compare/v3.8.0..v3.8.1 [v3.8.0]: https://github.com/sparkgeo/stac-validator/compare/v3.7.0..v3.8.0 [v3.7.0]: https://github.com/sparkgeo/stac-validator/compare/v3.6.0..v3.7.0 diff --git a/setup.py b/setup.py index 33aba059..c80bf1ed 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -__version__ = "3.8.1" +__version__ = "3.9.0" with open("README.md", "r") as fh: long_description = fh.read() From b6af30b2aa78a06f3a3a6c41cbc6dd9f29a68fc6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 16:14:48 +0800 Subject: [PATCH 02/10] improve error messaging --- CHANGELOG.md | 5 + setup.py | 2 +- stac_validator/validate.py | 47 +- tests/test_assets.py | 1 + tests/test_core.py | 1 + tests/test_custom.py | 2 + tests/test_data/v110/extended-item.json | 210 +++++++++ .../test_data/v110/test-sar-item-invalid.json | 415 ++++++++++++++++++ tests/test_default.py | 19 +- tests/test_header.py | 1 + tests/test_links.py | 1 + tests/test_recursion.py | 4 + 12 files changed, 689 insertions(+), 19 deletions(-) create mode 100644 tests/test_data/v110/extended-item.json create mode 100644 tests/test_data/v110/test-sar-item-invalid.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d46a10..a664f930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ## [Unreleased] +## [v3.9.1] - 2025-06-13 + +### Added +- `failed_schema` field to error messages for easier error handling +- `recommendation` field to --verbose mode ## [v3.9.0] - 2025-06-13 diff --git a/setup.py b/setup.py index c80bf1ed..27e5862e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -__version__ = "3.9.0" +__version__ = "3.9.1" with open("README.md", "r") as fh: long_description = fh.read() diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 2e4681e6..00b3b32d 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -267,21 +267,39 @@ def create_err_msg( schema_value = str(self.schema) schema_field: List[str] = [schema_value] if schema_value else [] + # Initialize the message with common fields message: Dict[str, Union[str, bool, List[str], Dict[str, Any]]] = { "version": version_str, "path": path_str, - "schema": schema_field, # Ensure schema is a list of strings or None + "schema": schema_field, # All schemas that were checked "valid_stac": False, "error_type": err_type, "error_message": err_msg, + "failed_schema": "", # Will be populated if we can determine which schema failed } + # Try to extract the failed schema from the error message if it's a validation error + if error_obj and hasattr(error_obj, "schema"): + if isinstance(error_obj.schema, dict) and "$id" in error_obj.schema: + message["failed_schema"] = error_obj.schema["$id"] + elif hasattr(error_obj, "schema_url"): + message["failed_schema"] = error_obj.schema_url + # If we can't find a schema ID, try to get it from the schema map + elif schema_field and len(schema_field) == 1: + message["failed_schema"] = schema_field[0] + if self.verbose and error_obj is not None: verbose_err = self._create_verbose_err_msg(error_obj) if isinstance(verbose_err, dict): message["error_verbose"] = verbose_err else: message["error_verbose"] = {"detail": str(verbose_err)} + # Add recommendation to check the schema if the error is not clear + if "failed_schema" in message and message["failed_schema"]: + message["recommendation"] = ( + "If the error is unclear, please check the schema documentation at: " + f"{message['failed_schema']}" + ) else: message["recommendation"] = ( "For more accurate error information, rerun with --verbose." @@ -459,13 +477,18 @@ def extensions_validator(self, stac_type: str) -> Dict: if e.context: e = best_match(e.context) # type: ignore valid = False - if e.absolute_path: - err_msg = ( - f"{e.message}. Error is in " - f"{' -> '.join(map(str, e.absolute_path))} " - ) - else: - err_msg = f"{e.message}" + # Get the current schema (extension) that caused the validation error + current_schema = self._original_schema_paths.get(extension, extension) + # Set the failed schema on the error object for create_err_msg to pick up + if not hasattr(e, "failed_schema"): + e.failed_schema = current_schema + # Build the error message with schema and path information + path_info = ( + f"Error is in {' -> '.join(map(str, e.absolute_path))} " + if e.absolute_path + else "" + ) + err_msg = f"{e.message}. {path_info}" message = self.create_err_msg( err_type="JSONSchemaValidationError", err_msg=err_msg, @@ -477,7 +500,13 @@ def extensions_validator(self, stac_type: str) -> Dict: if self.recursive: raise valid = False - err_msg = f"{e}. Error in Extensions." + # Include the current schema in the error message for other types of exceptions + current_schema = ( + self._original_schema_paths.get(extension, extension) + if "extension" in locals() + else "unknown schema" + ) + err_msg = f"{e} [Schema: {current_schema}]. Error in Extensions." return self.create_err_msg( err_type="Exception", err_msg=err_msg, error_obj=e ) diff --git a/tests/test_assets.py b/tests/test_assets.py index 1d2dc1e4..7734952b 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -22,6 +22,7 @@ def test_assets_v090(): ], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://cdn.staclint.com/v0.9.0/extension/view.json", "error_message": "-0.00751271 is less than the minimum of 0. Error is in properties -> view:off_nadir ", "recommendation": "For more accurate error information, rerun with --verbose.", "validation_method": "default", diff --git a/tests/test_core.py b/tests/test_core.py index 23ae487d..0e116b26 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -83,6 +83,7 @@ def test_core_bad_item_local_v090(): "schema": ["https://cdn.staclint.com/v0.9.0/item.json"], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://cdn.staclint.com/v0.9.0/item.json", "error_message": "'id' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", } diff --git a/tests/test_custom.py b/tests/test_custom.py index b9299cd4..de10e1e4 100644 --- a/tests/test_custom.py +++ b/tests/test_custom.py @@ -20,6 +20,7 @@ def test_custom_item_remote_schema_v080(): "validation_method": "custom", "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://cdn.staclint.com/v0.8.0/item.json", "error_message": "'bbox' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", } @@ -75,6 +76,7 @@ def test_custom_bad_item_remote_schema_v090(): "schema": ["https://cdn.staclint.com/v0.9.0/item.json"], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://cdn.staclint.com/v0.9.0/item.json", "error_message": "'id' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", } diff --git a/tests/test_data/v110/extended-item.json b/tests/test_data/v110/extended-item.json new file mode 100644 index 00000000..a6828421 --- /dev/null +++ b/tests/test_data/v110/extended-item.json @@ -0,0 +1,210 @@ +{ + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/remote-data/v1.0.0/schema.json" + ], + "type": "Feature", + "id": "20201211_223832_CS2", + "bbox": [ + 172.91173669923782, + 1.3438851951615003, + 172.95469614953714, + 1.3690476620161975 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 172.91173669923782, + 1.3438851951615003 + ], + [ + 172.95469614953714, + 1.3438851951615003 + ], + [ + 172.95469614953714, + 1.3690476620161975 + ], + [ + 172.91173669923782, + 1.3690476620161975 + ], + [ + 172.91173669923782, + 1.3438851951615003 + ] + ] + ] + }, + "properties": { + "title": "Extended Item", + "description": "A sample STAC Item that includes a variety of examples from the stable extensions", + "keywords": [ + "extended", + "example", + "item" + ], + "datetime": "2020-12-14T18:02:31.437000Z", + "created": "2020-12-15T01:48:13.725Z", + "updated": "2020-12-15T01:48:13.725Z", + "platform": "cool_sat2", + "instruments": [ + "cool_sensor_v2" + ], + "gsd": 0.66, + "eo:cloud_cover": 1.2, + "eo:snow_cover": 0, + "statistics": { + "vegetation": 12.57, + "water": 1.23, + "urban": 26.2 + }, + "proj:code": "EPSG:32659", + "proj:shape": [ + 5558, + 9559 + ], + "proj:transform": [ + 0.5, + 0, + 712710, + 0, + -0.5, + 151406, + 0, + 0, + 1 + ], + "view:sun_elevation": 54.9, + "view:off_nadir": 3.8, + "view:sun_azimuth": 135.7, + "rd:type": "scene", + "rd:anomalous_pixels": 0.14, + "rd:earth_sun_distance": 1.014156, + "rd:sat_id": "cool_sat2", + "rd:product_level": "LV3A", + "sci:doi": "10.5061/dryad.s2v81.2/27.2" + }, + "collection": "simple-collection", + "links": [ + { + "rel": "collection", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection" + }, + { + "rel": "root", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection" + }, + { + "rel": "parent", + "href": "./collection.json", + "type": "application/json", + "title": "Simple Example Collection" + }, + { + "rel": "alternate", + "type": "text/html", + "href": "http://remotedata.io/catalog/20201211_223832_CS2/index.html", + "title": "HTML version of this STAC Item" + } + ], + "assets": { + "analytic": { + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "4-Band Analytic", + "roles": [ + "data" + ], + "bands": [ + { + "name": "band1", + "eo:common_name": "blue", + "eo:center_wavelength": 0.47, + "eo:full_width_half_max": 70 + }, + { + "name": "band2", + "eo:common_name": "green", + "eo:center_wavelength": 0.56, + "eo:full_width_half_max": 80 + }, + { + "name": "band3", + "eo:common_name": "red", + "eo:center_wavelength": 0.645, + "eo:full_width_half_max": 90 + }, + { + "name": "band4", + "eo:common_name": "nir", + "eo:center_wavelength": 0.8, + "eo:full_width_half_max": 152 + } + ] + }, + "thumbnail": { + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg", + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ] + }, + "visual": { + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "3-Band Visual", + "roles": [ + "visual" + ], + "bands": [ + { + "name": "band3", + "eo:common_name": "red", + "eo:center_wavelength": 0.645, + "eo:full_width_half_max": 90 + }, + { + "name": "band2", + "eo:common_name": "green", + "eo:center_wavelength": 0.56, + "eo:full_width_half_max": 80 + }, + { + "name": "band1", + "eo:common_name": "blue", + "eo:center_wavelength": 0.47, + "eo:full_width_half_max": 70 + } + ] + }, + "udm": { + "href": "https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2_analytic_udm.tif", + "title": "Unusable Data Mask", + "type": "image/tiff; application=geotiff" + }, + "json-metadata": { + "href": "http://remotedata.io/catalog/20201211_223832_CS2/extended-metadata.json", + "title": "Extended Metadata", + "type": "application/json", + "roles": [ + "metadata" + ] + }, + "ephemeris": { + "href": "http://cool-sat.com/catalog/20201211_223832_CS2/20201211_223832_CS2.EPH", + "title": "Satellite Ephemeris Metadata" + } + } + } \ No newline at end of file diff --git a/tests/test_data/v110/test-sar-item-invalid.json b/tests/test_data/v110/test-sar-item-invalid.json new file mode 100644 index 00000000..9863e8d2 --- /dev/null +++ b/tests/test_data/v110/test-sar-item-invalid.json @@ -0,0 +1,415 @@ + +{ + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/product/v0.1.0/schema.json", + "https://stac-extensions.github.io/sar/v1.1.0/schema.json", + "https://stac-extensions.github.io/altimetry/v0.1.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/sat/v1.1.0/schema.json", + "https://stac-extensions.github.io/sentinel-1/v0.2.0/schema.json", + "https://stac-extensions.github.io/processing/v1.2.0/schema.json", + "https://stac-extensions.github.io/storage/v2.0.0/schema.json", + "https://stac-extensions.github.io/ceos-ard/v0.2.0/schema.json" + ], + "id": "OPERA_L2_RTC-S1_T070-149822-IW3_20220101T124811Z_20250611T234746Z_S1A_20_v0.1", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 164.5738105155539, + -77.09744736586842 + ], + [ + 164.5841107737331, + -77.06092164981267 + ], + [ + 164.5943055896268, + -77.02456861667501 + ], + [ + 164.6043976249054, + -76.98838437391879 + ], + [ + 164.6143886247731, + -76.95236515068486 + ], + [ + 164.6242810255355, + -76.91650731426346 + ], + [ + 164.6340769950861, + -76.88080735064491 + ], + [ + 164.6437788191498, + -76.84526186443392 + ], + [ + 164.6533879540521, + -76.80986756023115 + ], + [ + 164.6629066156633, + -76.77462126015926 + ], + [ + 164.6723366189091, + -76.73951988279993 + ], + [ + 164.6816798029899, + -76.70456044480147 + ], + [ + 164.6909381371846, + -76.66974005821108 + ], + [ + 164.7001127849088, + -76.63505591330626 + ], + [ + 164.7092056716351, + -76.60050529702578 + ], + [ + 164.7182183385663, + -76.56608557323291 + ], + [ + 164.7271523618029, + -76.53179418528892 + ], + [ + 164.7360094562833, + -76.49762865434306 + ], + [ + 164.7447905508336, + -76.46358656324253 + ], + [ + 164.7534973361782, + -76.42966557554502 + ], + [ + 164.7620626941835, + -76.39613193601812 + ], + [ + 163.9577752171824, + -76.3834733821606 + ], + [ + 163.9472896009472, + -76.41697662475501 + ], + [ + 163.9366312297678, + -76.45086671776188 + ], + [ + 163.9258819911961, + -76.48487764991665 + ], + [ + 163.9150401310552, + -76.51901175603307 + ], + [ + 163.9041039255227, + -76.55327144154155 + ], + [ + 163.8930712657604, + -76.58765917815546 + ], + [ + 163.8819407934205, + -76.62217752659461 + ], + [ + 163.8707103472533, + -76.65682911282163 + ], + [ + 163.8593778902213, + -76.69161664748461 + ], + [ + 163.8479411325865, + -76.7265429229195 + ], + [ + 163.8363984494731, + -76.7616108328091 + ], + [ + 163.8247474153511, + -76.79682335070363 + ], + [ + 163.8129857401177, + -76.83218355043469 + ], + [ + 163.8011110570616, + -76.86769460684839 + ], + [ + 163.7891207073698, + -76.90335979673557 + ], + [ + 163.7770126897196, + -76.93918252025401 + ], + [ + 163.7647841747692, + -76.97516628039226 + ], + [ + 163.7524324309662, + -77.01131470459156 + ], + [ + 163.7399544370051, + -77.04763154391978 + ], + [ + 163.7273478409169, + -77.08412069598076 + ], + [ + 164.5738105155539, + -77.09744736586842 + ] + ] + ] + }, + "bbox": [ + 163.7273478409169, + -77.09744736586842, + 164.7620626941835, + -76.3834733821606 + ], + "properties": { + "gsd": 20.0, + "constellation": "Sentinel-1", + "platform": "Sentinel-1A", + "instruments": [ + "Sentinel-1A CSAR" + ], + "created": "2025-06-11T23:47:50.172887Z", + "start_datetime": "2022-01-01T12:48:11.446000Z", + "end_datetime": "2022-01-01T12:48:14.539612Z", + "odc:product": "ga_s1_iw_hh_c1", + "odc:product_family": "sar_ard", + "odc:region_code": "t070_149822_iw3", + "product:type": "RTC_S1", + "ceosard:type": "radar", + "ceosard:specification": "NRB", + "ceosard:specification_version": "1.1", + "proj:code": "EPSG:3031", + "proj:bbox": [ + 368760.0, + -1438080.0, + 416760.0, + -1347300.0 + ], + "sar:frequency_band": "C", + "sar:center_frequency": 5405000454.33435, + "sar:polarizations": [ + "HH" + ], + "sar:observation_direction": "right", + "sar:relative_burst": "t070_149822_iw3", + "sar:beam_ids": "IW3", + "altm:instrument_type": "sar", + "altm:instrument_mode": "IW", + "sat:orbit_state": "descending", + "sat:absolute_orbit": 41267, + "sat:relative_orbit": 70, + "sat:orbit_cycle": "12", + "sat:osv": [ + "S1A_OPER_AUX_POEORB_OPOD_20220121T121549_V20211231T225942_20220102T005942.EOF" + ], + "sat:orbit_state_vectors": "TODO", + "s1:orbit_source": "POE precise orbit", + "processing:level": "L2", + "processing:facility": "Geoscience Australia", + "processing:datetime": "2025-06-11T23:47:50.172887Z", + "processing:version": 0.1, + "processing:software": { + "isce3": "0.15.0", + "s1Reader": "0.2.5", + "OPERA-adt/RTC": "1.0.4", + "sar-pipeline": "0.2.2b1.dev26+g043d201.d20250613", + "dem-handler": "0.2.2" + }, + "sarard:source_id": [ + "S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD.SAFE" + ], + "sarard:scene_id": "S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD", + "sarard:pixel_spacing_x": 20.0, + "sarard:pixel_spacing_y": 20.0, + "sarard:resolution_x": 20.0, + "sarard:resolution_y": 20.0, + "sarard:speckle_filter_applied": false, + "sarard:speckle_filter_type": "", + "sarard:speckle_filter_window": [], + "sarard:measurement_type": "gamma0", + "sarard:measurement_convention": "linear backscatter intensity", + "sarard:conversion_eq": "10*log10(backscatter_linear)", + "sarard:noise_removal_applied": true, + "sarard:static_tropospheric_correction_applied": true, + "sarard:wet_tropospheric_correction_applied": false, + "sarard:bistatic_correction_applied": true, + "sarard:ionospheric_correction_applied": false, + "sarard:geometric_accuracy_ALE": "TODO", + "sarard:geometric_accuracy_rmse": "TODO", + "sarard:geometric_accuracy_range": "TODO", + "sarard:geometric_accuracy_azimuth": "TODO", + "storage:schemes": { + "aws-std": { + "type": "aws-s3", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "deant-data-public-dev", + "region": "ap-southeast-2", + "requester_pays": true + } + }, + "datetime": "2022-01-01T12:48:11.446000Z" + }, + "links": [ + { + "rel": "ceos-ard-specification", + "href": "https://ceos.org/ard/files/PFS/SAR/v1.1/CEOS-ARD_PFS_Synthetic_Aperture_Radar_v1.1.pdf", + "type": "application/pdf" + }, + { + "rel": "geoid-source", + "href": "https://aria-geoid.s3.us-west-2.amazonaws.com/us_nga_egm2008_1_4326__agisoft.tif" + }, + { + "rel": "derived_from", + "href": "https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD.zip" + }, + { + "rel": "dem-source", + "href": "https://registry.opendata.aws/copernicus-dem/" + }, + { + "rel": "rtc-algorithm", + "href": "https://doi.org/10.1109/TGRS.2022.3147472" + }, + { + "rel": "geocoding-algorithm", + "href": "https://doi.org/10.1109/TGRS.2022.3147472" + }, + { + "rel": "noise-correction", + "href": "https://sentinels.copernicus.eu/documents/247904/2142675/Thermal-Denoising-of-Products-Generated-by-Sentinel-1-IPF.pdf" + }, + { + "rel": "additional-metadata", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/s1_rtc_c1/t070_149822_iw3/2022/1/1/OPERA_L2_RTC-S1_T070-149822-IW3_20220101T124811Z_20250611T234746Z_S1A_20_v0.1.h5" + }, + { + "rel": "self", + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/s1_rtc_c1/t070_149822_iw3/2022/1/1/metadata.json", + "type": "application/json" + }, + { + "rel": "collection", + "href": "./collection.json", + "type": "application/json" + } + ], + "assets": { + "HH": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/s1_rtc_c1/t070_149822_iw3/2022/1/1/OPERA_L2_RTC-S1_T070-149822-IW3_20220101T124811Z_20250611T234746Z_S1A_20_v0.1_HH.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "HH", + "description": "HH polarised backscatter", + "proj:shape": [ + 4539, + 2400 + ], + "proj:transform": [ + 20.0, + 0.0, + 368760.0, + 0.0, + -20.0, + -1347300.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:3031", + "raster:data_type": "float32", + "raster:sampling": "Area", + "raster:nodata": "nan", + "processing:level": "L2", + "roles": [ + "data", + "backscatter" + ] + }, + "mask": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/s1_rtc_c1/t070_149822_iw3/2022/1/1/OPERA_L2_RTC-S1_T070-149822-IW3_20220101T124811Z_20250611T234746Z_S1A_20_v0.1_mask.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "mask", + "description": "shadow layover data mask", + "proj:shape": [ + 4539, + 2400 + ], + "proj:transform": [ + 20.0, + 0.0, + 368760.0, + 0.0, + -20.0, + -1347300.0, + 0.0, + 0.0, + 1.0 + ], + "proj:code": "EPSG:3031", + "raster:data_type": "uint8", + "raster:sampling": "Area", + "raster:nodata": 255.0, + "raster:values": { + "shadow": 1, + "layover": 2, + "shadow_and_layover": 3, + "invalid_sample": 255 + }, + "roles": [ + "data", + "auxiliary", + "mask", + "shadow", + "layover" + ] + }, + "thumbnail": { + "href": "https://deant-data-public-dev.s3.ap-southeast-2.amazonaws.com/experimental/s1_rtc_c1/t070_149822_iw3/2022/1/1/OPERA_L2_RTC-S1_T070-149822-IW3_20220101T124811Z_20250611T234746Z_S1A_20_v0.1.png", + "type": "image/png", + "title": "thumbnail", + "description": "thumbnail image for backscatter", + "roles": [ + "thumbnail" + ] + } + }, + "collection": "s1_rtc_c1" +} diff --git a/tests/test_default.py b/tests/test_default.py index 4b7316a7..44c6e13c 100644 --- a/tests/test_default.py +++ b/tests/test_default.py @@ -3,8 +3,6 @@ """ -import pytest - from stac_validator import stac_validator @@ -24,18 +22,21 @@ def test_default_v070(): ] -@pytest.mark.skip(reason="staclint eo extension schema invalid") -def test_default_item_local_v080(): - stac_file = "tests/test_data/v080/items/sample-full.json" +def test_default_item_local_v110(): + stac_file = "tests/test_data/v110/extended-item.json" stac = stac_validator.StacValidate(stac_file) stac.run() assert stac.message == [ { - "version": "0.8.0", - "path": "tests/test_data/v080/items/sample-full.json", + "version": "1.1.0", + "path": "tests/test_data/v110/extended-item.json", "schema": [ - "https://cdn.staclint.com/v0.8.0/extension/eo.json", - "https://cdn.staclint.com/v0.8.0/item.json", + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/remote-data/v1.0.0/schema.json", + "https://schemas.stacspec.org/v1.1.0/item-spec/json-schema/item.json", ], "asset_type": "ITEM", "validation_method": "default", diff --git a/tests/test_header.py b/tests/test_header.py index ada552c3..6203bd05 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -45,6 +45,7 @@ def test_header(): "schema": [], "valid_stac": False, "error_type": "HTTPError", + "failed_schema": "", "error_message": "403 Client Error: None for url: https://localhost/tests/test_data/v110/simple-item.json", "recommendation": "For more accurate error information, rerun with --verbose.", } diff --git a/tests/test_links.py b/tests/test_links.py index 2944d2d1..913bd135 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -20,6 +20,7 @@ def test_poorly_formatted_v090(): ], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://cdn.staclint.com/v0.9.0/extension/view.json", "error_message": "-0.00751271 is less than the minimum of 0. Error is in properties -> view:off_nadir ", "recommendation": "For more accurate error information, rerun with --verbose.", "validation_method": "default", diff --git a/tests/test_recursion.py b/tests/test_recursion.py index 7a26bfd0..6f308bb5 100644 --- a/tests/test_recursion.py +++ b/tests/test_recursion.py @@ -321,6 +321,7 @@ def test_recursion_with_bad_item(): ], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json", "error_message": "'id' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", }, @@ -352,6 +353,7 @@ def test_recursion_with_bad_item_trace_recursion(): ], "valid_stac": False, "error_type": "JSONSchemaValidationError", + "failed_schema": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json", "error_message": "'id' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", }, @@ -378,6 +380,7 @@ def test_recursion_with_bad_child_collection(): "asset_type": "COLLECTION", "validation_method": "recursive", "error_type": "JSONSchemaValidationError", + "failed_schema": "https://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json", "error_message": "'id' is a required property", "recommendation": "For more accurate error information, rerun with --verbose.", } @@ -401,6 +404,7 @@ def test_recursion_with_missing_collection_link(): "valid_stac": False, "validation_method": "recursive", "error_type": "JSONSchemaValidationError", + "failed_schema": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json", "error_message": "'simple-collection' should not be valid under {}. Error is in collection ", "recommendation": "For more accurate error information, rerun with --verbose.", }, From 823e02086b686a47ddfeb9707929403a32e57329 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:06:00 +0800 Subject: [PATCH 03/10] mypy, changelog --- CHANGELOG.md | 4 ++-- stac_validator/validate.py | 26 +++++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a664f930..e6a32ed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ## [v3.9.1] - 2025-06-13 ### Added -- `failed_schema` field to error messages for easier error handling -- `recommendation` field to --verbose mode +- `failed_schema` field to error messages for easier error handling [#260](https://github.com/stac-utils/stac-validator/pull/260) +- `recommendation` field to --verbose mode [#260](https://github.com/stac-utils/stac-validator/pull/260) ## [v3.9.0] - 2025-06-13 diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 00b3b32d..26390ca8 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -478,10 +478,7 @@ def extensions_validator(self, stac_type: str) -> Dict: e = best_match(e.context) # type: ignore valid = False # Get the current schema (extension) that caused the validation error - current_schema = self._original_schema_paths.get(extension, extension) - # Set the failed schema on the error object for create_err_msg to pick up - if not hasattr(e, "failed_schema"): - e.failed_schema = current_schema + failed_schema = self._original_schema_paths.get(extension, extension) # Build the error message with schema and path information path_info = ( f"Error is in {' -> '.join(map(str, e.absolute_path))} " @@ -489,11 +486,30 @@ def extensions_validator(self, stac_type: str) -> Dict: else "" ) err_msg = f"{e.message}. {path_info}" + # Create a new error object with the original message + error_with_schema = type(verbose_error)( + message=verbose_error.message, + validator=verbose_error.validator, + path=list(verbose_error.path), + cause=verbose_error.cause, + context=list(verbose_error.context) if verbose_error.context else [], + validator_value=verbose_error.validator_value, + instance=verbose_error.instance, + schema=verbose_error.schema, + schema_path=list(verbose_error.schema_path), + parent=verbose_error.parent, + ) + + # Create the error message with the original format message = self.create_err_msg( err_type="JSONSchemaValidationError", err_msg=err_msg, - error_obj=verbose_error, + error_obj=error_with_schema, ) + + # Set the failed_schema in the message if we have it + if failed_schema: + message["failed_schema"] = failed_schema return message except Exception as e: From d41fd3421433333573048dcb838184203c846b31 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:14:43 +0800 Subject: [PATCH 04/10] mypy --- setup.py | 1 - stac_validator/validate.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6fb7040b..27e5862e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ from setuptools import setup - __version__ = "3.9.1" with open("README.md", "r") as fh: diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 26390ca8..a91c0e1d 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -489,7 +489,11 @@ def extensions_validator(self, stac_type: str) -> Dict: # Create a new error object with the original message error_with_schema = type(verbose_error)( message=verbose_error.message, - validator=verbose_error.validator, + validator=( + verbose_error.validator.__name__ + if verbose_error.validator + else None + ), path=list(verbose_error.path), cause=verbose_error.cause, context=list(verbose_error.context) if verbose_error.context else [], From 9b36e97ec5ed07fdd607d3c46ad2ba7f51bddf11 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:18:44 +0800 Subject: [PATCH 05/10] handle validator types --- stac_validator/validate.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/stac_validator/validate.py b/stac_validator/validate.py index a91c0e1d..06194d7f 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -487,13 +487,19 @@ def extensions_validator(self, stac_type: str) -> Dict: ) err_msg = f"{e.message}. {path_info}" # Create a new error object with the original message + validator_name = None + if verbose_error.validator is not None: + # Handle different possible validator types + if hasattr(verbose_error.validator, '__name__'): + validator_name = verbose_error.validator.__name__ + elif hasattr(verbose_error.validator, '__class__') and hasattr(verbose_error.validator.__class__, '__name__'): + validator_name = verbose_error.validator.__class__.__name__ + elif isinstance(verbose_error.validator, str): + validator_name = verbose_error.validator + error_with_schema = type(verbose_error)( message=verbose_error.message, - validator=( - verbose_error.validator.__name__ - if verbose_error.validator - else None - ), + validator=validator_name, path=list(verbose_error.path), cause=verbose_error.cause, context=list(verbose_error.context) if verbose_error.context else [], From 9d7683135f44a28344db399ce935b98c4dcadcac Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:19:08 +0800 Subject: [PATCH 06/10] pre-commit --- stac_validator/validate.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 06194d7f..056074e3 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -490,13 +490,15 @@ def extensions_validator(self, stac_type: str) -> Dict: validator_name = None if verbose_error.validator is not None: # Handle different possible validator types - if hasattr(verbose_error.validator, '__name__'): + if hasattr(verbose_error.validator, "__name__"): validator_name = verbose_error.validator.__name__ - elif hasattr(verbose_error.validator, '__class__') and hasattr(verbose_error.validator.__class__, '__name__'): + elif hasattr(verbose_error.validator, "__class__") and hasattr( + verbose_error.validator.__class__, "__name__" + ): validator_name = verbose_error.validator.__class__.__name__ elif isinstance(verbose_error.validator, str): validator_name = verbose_error.validator - + error_with_schema = type(verbose_error)( message=verbose_error.message, validator=validator_name, From c2629d8010474dff760063087dbe5bf05231b8aa Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:30:24 +0800 Subject: [PATCH 07/10] run mypy in 3.12 --- .github/workflows/test-runner.yml | 3 ++- stac_validator/validate.py | 18 ++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b7f4cf71..7be3c3ed 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -26,10 +26,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run mypy + if: matrix.python-version == '3.12' run: | pip install . pip install -r requirements-dev.txt - pytest --mypy stac_validator + mypy stac_validator/ - name: Run pre-commit if: matrix.python-version == 3.12 diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 056074e3..c944c57a 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -479,29 +479,19 @@ def extensions_validator(self, stac_type: str) -> Dict: valid = False # Get the current schema (extension) that caused the validation error failed_schema = self._original_schema_paths.get(extension, extension) - # Build the error message with schema and path information + # Build the error message with path information path_info = ( f"Error is in {' -> '.join(map(str, e.absolute_path))} " if e.absolute_path else "" ) err_msg = f"{e.message}. {path_info}" - # Create a new error object with the original message - validator_name = None - if verbose_error.validator is not None: - # Handle different possible validator types - if hasattr(verbose_error.validator, "__name__"): - validator_name = verbose_error.validator.__name__ - elif hasattr(verbose_error.validator, "__class__") and hasattr( - verbose_error.validator.__class__, "__name__" - ): - validator_name = verbose_error.validator.__class__.__name__ - elif isinstance(verbose_error.validator, str): - validator_name = verbose_error.validator + # Create a new error object with the original message + # We'll keep the original validator object to satisfy type checking error_with_schema = type(verbose_error)( message=verbose_error.message, - validator=validator_name, + validator=verbose_error.validator, # Keep the original validator object path=list(verbose_error.path), cause=verbose_error.cause, context=list(verbose_error.context) if verbose_error.context else [], From f77df5b49ee7aeede4ebe382745e5c8df5a09b86 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 18:48:14 +0800 Subject: [PATCH 08/10] fix --- stac_validator/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_validator/validate.py b/stac_validator/validate.py index c944c57a..0e34ee88 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -491,7 +491,7 @@ def extensions_validator(self, stac_type: str) -> Dict: # We'll keep the original validator object to satisfy type checking error_with_schema = type(verbose_error)( message=verbose_error.message, - validator=verbose_error.validator, # Keep the original validator object + validator=verbose_error.validator, # type: ignore[arg-type] # Keep the original validator object path=list(verbose_error.path), cause=verbose_error.cause, context=list(verbose_error.context) if verbose_error.context else [], From 4f1cf72a3190c1ea4a3d82a04521d389c8900243 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 19:13:42 +0800 Subject: [PATCH 09/10] clean up --- stac_validator/validate.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/stac_validator/validate.py b/stac_validator/validate.py index 0e34ee88..bec8c426 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -487,26 +487,11 @@ def extensions_validator(self, stac_type: str) -> Dict: ) err_msg = f"{e.message}. {path_info}" - # Create a new error object with the original message - # We'll keep the original validator object to satisfy type checking - error_with_schema = type(verbose_error)( - message=verbose_error.message, - validator=verbose_error.validator, # type: ignore[arg-type] # Keep the original validator object - path=list(verbose_error.path), - cause=verbose_error.cause, - context=list(verbose_error.context) if verbose_error.context else [], - validator_value=verbose_error.validator_value, - instance=verbose_error.instance, - schema=verbose_error.schema, - schema_path=list(verbose_error.schema_path), - parent=verbose_error.parent, - ) - - # Create the error message with the original format + # Create the error message with the original error object message = self.create_err_msg( err_type="JSONSchemaValidationError", err_msg=err_msg, - error_obj=error_with_schema, + error_obj=verbose_error, ) # Set the failed_schema in the message if we have it From d75e6a3c59ee6aac50e6977fae159567c939ff8b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 13 Jun 2025 19:19:45 +0800 Subject: [PATCH 10/10] update 3.9.1 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f632c0b6..f68d8a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,7 +272,8 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) - With the newest version - 1.0.0-beta.2 - items will run through jsonchema validation before the PySTAC validation. The reason for this is that jsonschema will give more informative error messages. This should be addressed better in the future. This is not the case with the --recursive option as time can be a concern here with larger collections. - Logging. Various additions were made here depending on the options selected. This was done to help assist people to update their STAC collections. -[Unreleased]: https://github.com/sparkgeo/stac-validator/compare/v3.9.0..main +[Unreleased]: https://github.com/sparkgeo/stac-validator/compare/v3.9.1..main +[v3.9.1]: https://github.com/sparkgeo/stac-validator/compare/v3.9.0..v3.9.1 [v3.9.0]: https://github.com/sparkgeo/stac-validator/compare/v3.8.1..v3.9.0 [v3.8.1]: https://github.com/sparkgeo/stac-validator/compare/v3.8.0..v3.8.1 [v3.8.0]: https://github.com/sparkgeo/stac-validator/compare/v3.7.0..v3.8.0