-
-
Notifications
You must be signed in to change notification settings - Fork 427
temporal: t.rast.aggregate: add -e flag to extend existing STRDS #7223
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
3c12011
eb11665
9b2110e
a39bdee
3f76909
e0de432
6ec7ac4
ca3f87e
d878245
1ca0f6e
d73d2a5
36700a4
75d75b0
0d65b33
58f6d4b
7e3d570
08a838c
5362536
201bbd7
7260037
215f13b
29414ef
c731f6f
eb21643
ea4d6f8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,10 +18,22 @@ | |
| :authors: Soeren Gebbert | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from typing import TYPE_CHECKING | ||
|
|
||
| from .abstract_map_dataset import AbstractMapDataset | ||
| from .core import get_current_mapset, get_tgis_message_interface, init_dbif | ||
| from .core import ( | ||
| SQLDatabaseInterfaceConnection, | ||
| get_current_mapset, | ||
| get_tgis_message_interface, | ||
| init_dbif, | ||
| ) | ||
| from .factory import dataset_factory | ||
|
|
||
| if TYPE_CHECKING: | ||
| from .abstract_space_time_dataset import AbstractSpaceTimeDataset | ||
|
|
||
| ############################################################################### | ||
|
|
||
|
|
||
|
|
@@ -104,7 +116,6 @@ def check_new_stds(name, type, dbif=None, overwrite: bool = False): | |
| """ | ||
|
|
||
| # Get the current mapset to create the id of the space time dataset | ||
|
|
||
| mapset = get_current_mapset() | ||
| msgr = get_tgis_message_interface() | ||
|
|
||
|
|
@@ -135,14 +146,191 @@ def check_new_stds(name, type, dbif=None, overwrite: bool = False): | |
|
|
||
| dbif, connection_state_changed = init_dbif(dbif) | ||
|
|
||
| if sp.is_in_db(dbif) and overwrite is False: | ||
| if sp.is_in_db(dbif) and not overwrite: | ||
| msgr.fatal( | ||
| _( | ||
| "Space time %(sp)s dataset <%(name)s> is already in the" | ||
| " database. Use the overwrite flag." | ||
| ) | ||
| % {"sp": sp.get_new_map_instance(None).get_type(), "name": name} | ||
| ) | ||
|
|
||
| if connection_state_changed: | ||
| dbif.close() | ||
|
|
||
| return sp | ||
|
|
||
|
|
||
| ############################################################################### | ||
|
|
||
|
|
||
| def _get_stds(stds_id: str, stds_type: str) -> AbstractSpaceTimeDataset: | ||
| """Return an initialized AbstractSpaceTimeDataset (STDS). | ||
|
|
||
| :param stds_id: The id of the space time dataset (name@mapset) | ||
| :param stds_type: The type of the space time dataset (strds, str3ds, stvds, | ||
| raster, vector, raster3d) | ||
|
|
||
| :return: An AbstractSpaceTimeDataset object | ||
|
|
||
| This function will raise a FatalError in case of an unknown type. | ||
| """ | ||
| msgr = get_tgis_message_interface() | ||
| supported_stds_types = { | ||
| "strds": "strds", | ||
| "rast": "strds", | ||
| "raster": "strds", | ||
| "str3ds": "str3ds", | ||
| "raster3d": "str3ds", | ||
| "rast3d": "str3ds", | ||
| "raster_3d": "str3ds", | ||
| "stvds": "stvds", | ||
| "vect": "stvds", | ||
| "vector": "stvds", | ||
| } | ||
| if stds_type not in supported_stds_types: | ||
| msgr.fatal(_("Unknown type: %s") % (stds_type)) | ||
| return dataset_factory(supported_stds_types[stds_type], stds_id) | ||
|
|
||
|
|
||
| ############################################################################### | ||
|
|
||
|
|
||
| def _ensure_id(name: str) -> str: | ||
| """Return a fully qualified id (name@mapset) for the given name. | ||
|
|
||
| If the name already contains a mapset, it is returned as is. | ||
| Otherwise, the current mapset is appended. | ||
|
|
||
| :param name: The name of the space time dataset | ||
|
|
||
| :return: The fully qualified id (name@mapset) | ||
| """ | ||
| if "@" not in name: | ||
| return f"{name}@{get_current_mapset()}" | ||
| return name | ||
|
|
||
|
|
||
| ############################################################################### | ||
|
|
||
|
|
||
| def check_open_output_stds( | ||
| name: str, | ||
| type: str, | ||
| dbif: SQLDatabaseInterfaceConnection | None = None, | ||
| overwrite: bool = False, | ||
| extend: bool = False, | ||
| ) -> bool: | ||
| """Check if an output space time dataset can be opened as requested. | ||
|
|
||
| This is a lightweight check run before main processing starts. | ||
| It does not create or open the dataset, only validates the request. | ||
|
|
||
| :param name: The name of the output space time dataset | ||
| :param type: The type of the space time dataset (strds, str3ds, stvds) | ||
| :param dbif: The temporal database interface to be used | ||
| :param overwrite: Flag to allow overwriting existing dataset | ||
| :param extend: Flag to extend an existing dataset if it exists | ||
| :return: True if the output dataset already exists, False otherwise | ||
|
|
||
| This function will raise a FatalError in case of an error. | ||
| """ | ||
| msgr = get_tgis_message_interface() | ||
|
|
||
| id = _ensure_id(name) | ||
| sp = _get_stds(id, type) | ||
|
|
||
| dbif, connection_state_changed = init_dbif(dbif) | ||
| output_exists = sp.is_in_db(dbif) | ||
|
|
||
| if output_exists and not extend and not overwrite: | ||
| if connection_state_changed: | ||
| dbif.close() | ||
| msgr.fatal( | ||
| _( | ||
| "Space time %(sp)s dataset <%(name)s> is already in the" | ||
| " database. Use the overwrite flag." | ||
| ) | ||
| % {"sp": sp.get_new_map_instance(None).get_type(), "name": name} | ||
| ) | ||
|
|
||
| if connection_state_changed: | ||
| dbif.close() | ||
|
|
||
| return output_exists | ||
|
|
||
|
|
||
| ############################################################################### | ||
|
|
||
|
|
||
| def open_output_stds( | ||
| name: str, | ||
| type: str, | ||
| temporaltype: str, | ||
| title: str, | ||
| descr: str, | ||
| semantic: str, | ||
| dbif: SQLDatabaseInterfaceConnection | None = None, | ||
| overwrite: bool = False, | ||
| extend: bool = False, | ||
| ): | ||
ninsbl marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| """Open or create an output space time dataset after main processing. | ||
|
|
||
| If extend is True and the dataset exists, opens and returns it. | ||
| Otherwise creates a new dataset (deleting existing if overwrite is True). | ||
|
|
||
| :param name: The name of the output space time dataset | ||
| :param type: The type of the space time dataset (strds, str3ds, stvds) | ||
| :param temporaltype: The temporal type (relative or absolute) | ||
| :param title: The title | ||
| :param descr: The dataset description | ||
| :param semantic: Semantical information | ||
| :param dbif: The temporal database interface to be used | ||
| :param overwrite: Flag to allow overwriting | ||
| :param extend: Flag to extend an existing dataset if it exists | ||
|
|
||
| :return: The opened or created space time dataset | ||
| """ | ||
| dbif, connection_state_changed = init_dbif(dbif) | ||
| msgr = get_tgis_message_interface() | ||
|
|
||
| id = _ensure_id(name) | ||
| sp = _get_stds(id, type) | ||
|
|
||
| if extend and sp.is_in_db(dbif): | ||
| # extend-if-exists: load and return existing dataset | ||
| sp.select(dbif) | ||
| if connection_state_changed: | ||
| dbif.close() | ||
| return sp | ||
|
Comment on lines
+294
to
+305
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You duplicate quite a bit of code... You may consider adding a helper function similar to e.g. (untested, so not suitable for copy and paste!):
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added _get_stds helper function to avoid repeating the type mapping block, and ensure_id helper to avoid repeating the id construction pattern. Kept them private ( prefix) since they're only used internally in open_stds.py happy to make them public if needed. |
||
|
|
||
| # Create new or overwrite existing | ||
| if sp.is_in_db(dbif): | ||
| msgr.warning( | ||
| _( | ||
| "Overwriting space time %(sp)s dataset <%(name)s> and " | ||
| "unregistering all maps" | ||
| ) | ||
| % {"sp": sp.get_new_map_instance(None).get_type(), "name": name} | ||
| ) | ||
| id = sp.get_id() | ||
| sp.delete(dbif) | ||
| sp = sp.get_new_instance(id) | ||
|
|
||
| msgr.verbose( | ||
| _("Creating a new space time %s dataset") | ||
| % sp.get_new_map_instance(None).get_type() | ||
| ) | ||
|
|
||
| sp.set_initial_values( | ||
| temporal_type=temporaltype, | ||
| semantic_type=semantic, | ||
| title=title, | ||
| description=descr, | ||
| ) | ||
|
|
||
| sp.insert(dbif) | ||
|
|
||
| if connection_state_changed: | ||
| dbif.close() | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -99,6 +99,11 @@ | |
| # %option G_OPT_T_WHERE | ||
| # %end | ||
|
|
||
| # %flag | ||
| # % key: e | ||
| # % description: Extend existing space time raster dataset | ||
| # %end | ||
|
|
||
| # %flag | ||
| # % key: n | ||
| # % description: Register Null maps | ||
|
|
@@ -120,6 +125,8 @@ def main(): | |
| gran = options["granularity"] | ||
| base = options["basename"] | ||
| register_null = flags["n"] | ||
| extend = flags["e"] | ||
| overwrite = gs.overwrite() | ||
| method = options["method"] | ||
| sampling = options["sampling"] | ||
| offset = options["offset"] | ||
|
|
@@ -144,8 +151,11 @@ def main(): | |
| dbif.close() | ||
| gs.fatal(_("Space time raster dataset <%s> is empty") % input) | ||
|
|
||
| # We will create the strds later, but need to check here | ||
| tgis.check_new_stds(output, "strds", dbif, gs.overwrite()) | ||
| # Check if output can be opened as requested before processing starts. | ||
| # This is a lightweight check that does not create or open the dataset. | ||
| output_exists = tgis.check_open_output_stds( | ||
| output, "strds", dbif=dbif, overwrite=overwrite, extend=extend | ||
| ) | ||
|
|
||
| start_time = map_list[0].temporal_extent.get_start_time() | ||
|
|
||
|
|
@@ -195,22 +205,24 @@ def main(): | |
| method=method, | ||
| nprocs=nprocs, | ||
| spatial=None, | ||
| overwrite=gs.overwrite(), | ||
| overwrite=overwrite, | ||
| file_limit=file_limit, | ||
| ) | ||
|
|
||
| if output_list: | ||
| # Open or create the output STRDS after processing succeeds | ||
| temporal_type, semantic_type, title, description = sp.get_initial_values() | ||
| output_strds = tgis.open_new_stds( | ||
| output_strds = tgis.open_output_stds( | ||
| output, | ||
| "strds", | ||
| temporal_type, | ||
| title, | ||
| description, | ||
| semantic_type, | ||
| dbif, | ||
| gs.overwrite(), | ||
| overwrite, | ||
| extend, | ||
| ) | ||
|
|
||
| register_null = not register_null | ||
|
|
||
| tgis.register_map_object_list( | ||
|
|
@@ -221,11 +233,17 @@ def main(): | |
| sp.get_relative_time_unit(), | ||
| dbif, | ||
| ) | ||
|
|
||
| # Refresh STRDS object to reflect newly registered maps | ||
| output_strds.select(dbif) | ||
|
Comment on lines
+236
to
+237
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is the reason to add this here? Was it a necessary change? And what does it accomplish?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, it was necessary to fix a bug. Without it, after register_map_object_list adds the new maps into the existing STRDS, the in-memory object still holds the old state (3 maps). select(dbif) refreshes it from the database so metadata.update records the correct map count (6). Removing it causes the extend test to fail. |
||
| # Update the raster metadata table entries with aggregation type | ||
| output_strds.set_aggregation_type(method) | ||
| output_strds.metadata.update(dbif) | ||
|
|
||
| # Update the command history. When extending, only update if the | ||
| # command string is not already recorded to avoid duplication. | ||
| if not output_exists or extend: | ||
| output_strds.update_command_string(dbif=dbif) | ||
|
|
||
| dbif.close() | ||
|
|
||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.