Merge pull request #673 from GIScience/get_template

refactor(indicators): separate templates from metadata and merge all metadata files

Author: matthiasschaub

CHANGELOG.md

@@ -9,9 +9,10 @@
 ### Other Changes
 
 - a new regression test suite has been added to support safer deployments of new versions ([#820])
+- refactor(indicators): separate templates from metadata and merge all metadata files ([#673])
 - use Pydantic model from the `geojson-pydantic` library as request model for `bpolys` ([#824])
 
-
+[#673]: https://github.com/GIScience/ohsome-quality-api/issues/673
 [#818]: https://github.com/GIScience/ohsome-quality-api/pull/818
 [#820]: https://github.com/GIScience/ohsome-quality-api/issues/820
 [#824]: https://github.com/GIScience/ohsome-quality-api/issues/824

ohsome_quality_api/api/api.py

@@ -42,14 +42,12 @@
 )
 from ohsome_quality_api.attributes.definitions import get_attributes, load_attributes
 from ohsome_quality_api.config import configure_logging
-from ohsome_quality_api.definitions import (
-    ATTRIBUTION_URL,
-    get_metadata,
-)
+from ohsome_quality_api.definitions import ATTRIBUTION_URL
 from ohsome_quality_api.indicators.definitions import (
     IndicatorEnum,
     IndicatorEnumRequest,
     get_coverage,
+    get_indicator,
     get_indicator_metadata,
 )
 from ohsome_quality_api.projects.definitions import (
@@ -75,7 +73,6 @@
 )
 from ohsome_quality_api.utils.helper import (
     get_class_from_key,
-    hyphen_to_camel,
     json_serialize,
 )
 from ohsome_quality_api.utils.validators import (
@@ -450,7 +447,7 @@ async def metadata_indicators(project: ProjectEnum = DEFAULT_PROJECT) -> Any:
 )
 async def metadata_indicators_by_key(key: IndicatorEnum) -> Any:
     """Get metadata of an indicator by key."""
-    metadata = get_metadata("indicators", hyphen_to_camel(key.value))
+    metadata = get_indicator(key.value)
     return {"result": {key.value: metadata}}
 
 

ohsome_quality_api/definitions.py

@@ -1,17 +1,14 @@
 """Global Variables and Functions."""
 
 import glob
-import logging
 from enum import Enum
 from types import MappingProxyType
-from typing import Iterable, Literal
+from typing import Literal
 
 import yaml
 
 from ohsome_quality_api.indicators.models import IndicatorMetadata
-from ohsome_quality_api.topics.definitions import load_topic_presets
 from ohsome_quality_api.utils.helper import (
-    camel_to_hyphen,
     get_module_dir,
 )
 
@@ -77,31 +74,6 @@ def load_metadata(
     return metadata
 
 
-def get_metadata(
-    module_name: Literal["indicators"], class_name: str
-) -> IndicatorMetadata:
-    """Get metadata of an indicator based on its class name.
-
-    This is implemented outside the metadata class to be able to access metadata of all
-    indicators without instantiating of those.
-
-    Args:
-        module_name: indicators.
-        class_name: Class name of an indicator (camel case).
-    """
-    metadata = load_metadata(module_name)
-    try:
-        return metadata[camel_to_hyphen(class_name)]
-    except KeyError:
-        logging.error("Invalid class name: " + class_name)
-        raise
-
-
-# TODO: duplicate of func with the same name in projects/definition.py ?
-def get_project_keys() -> Iterable[str]:
-    return set(t.project for t in load_topic_presets().values())
-
-
 def get_attribution(data_keys: list) -> str:
     """Return attribution text. Individual attributions are separated by semicolons."""
     assert set(data_keys) <= {"OSM", "GHSL", "VNL", "EUBUCCO", "Microsoft Buildings"}

ohsome_quality_api/indicators/attribute_completeness/indicator.py

@@ -67,7 +67,7 @@ def calculate(self) -> None:
             self.result.value = None
         if self.result.value is None:
             return
-        description = Template(self.metadata.result_description).substitute(
+        description = Template(self.templates.result_description).substitute(
             result=round(self.result.value, 2),
             all=round(self.absolute_value_1, 1),
             matched=round(self.absolute_value_2, 1),
@@ -79,17 +79,17 @@ def calculate(self) -> None:
         if self.result.value >= self.threshold_yellow:
             self.result.class_ = 5
             self.result.description = (
-                description + self.metadata.label_description["green"]
+                description + self.templates.label_description["green"]
             )
         elif self.threshold_yellow > self.result.value >= self.threshold_red:
             self.result.class_ = 3
             self.result.description = (
-                description + self.metadata.label_description["yellow"]
+                description + self.templates.label_description["yellow"]
             )
         else:
             self.result.class_ = 1
             self.result.description = (
-                description + self.metadata.label_description["red"]
+                description + self.templates.label_description["red"]
             )
 
     def create_figure(self) -> None:

ohsome_quality_api/indicators/attribute_completeness/metadata.yaml

@@ -0,0 +1,13 @@
+---
+label_description:
+  red: >-
+    Less than 25% of the features match the expected tags.
+  yellow: >-
+    Around 25-75% of the features match the expected tags.
+  green: >-
+    More than 75% of the features match the expected tags.
+  undefined: >-
+    The quality level could not be calculated for this indicator.
+result_description: >-
+  The ratio of the features (all: $all) compared to features with
+  expected tags (matched: $matched) is $result. 

ohsome_quality_api/indicators/attribute_completeness/templates.yaml

@@ -1,13 +1,25 @@
 import json
+import os
 from abc import ABCMeta, abstractmethod
 
 import plotly.graph_objects as go
+import yaml
 from geojson import Feature, Polygon
 
-from ohsome_quality_api.definitions import get_attribution, get_metadata
-from ohsome_quality_api.indicators.models import IndicatorMetadata, Result
+from ohsome_quality_api.definitions import get_attribution
+from ohsome_quality_api.indicators.definitions import get_indicator
+from ohsome_quality_api.indicators.models import (
+    IndicatorMetadata,
+    IndicatorTemplates,
+    Result,
+)
 from ohsome_quality_api.topics.models import BaseTopic as Topic
-from ohsome_quality_api.utils.helper import json_serialize
+from ohsome_quality_api.utils.helper import (
+    camel_to_hyphen,
+    camel_to_snake,
+    get_module_dir,
+    json_serialize,
+)
 
 
 class BaseIndicator(metaclass=ABCMeta):
@@ -18,13 +30,14 @@ def __init__(
         topic: Topic,
         feature: Feature,
     ) -> None:
-        self.metadata: IndicatorMetadata = get_metadata(
-            "indicators", type(self).__name__
+        self.metadata: IndicatorMetadata = get_indicator(
+            camel_to_hyphen(type(self).__name__)
         )
+        self.templates: IndicatorTemplates = self.get_template()
         self.topic: Topic = topic
         self.feature: Feature = feature
         self.result: Result = Result(
-            description=self.metadata.label_description["undefined"],
+            description=self.templates.label_description["undefined"],
         )
         self._get_default_figure()
 
@@ -34,10 +47,7 @@ def as_dict(self, include_data: bool = False, exclude_label: bool = False) -> di
         else:
             result = self.result.model_dump(by_alias=True)
         raw_dict = {
-            "metadata": self.metadata.model_dump(
-                by_alias=True,
-                exclude={"result_description", "label_description"},
-            ),
+            "metadata": self.metadata.model_dump(by_alias=True),
             "topic": self.topic.model_dump(
                 by_alias=True,
                 exclude={"ratio_filter"},
@@ -86,6 +96,7 @@ def data(self) -> dict:
         data = vars(self).copy()
         data.pop("result")
         data.pop("metadata")
+        data.pop("templates")
         data.pop("topic")
         data.pop("feature")
         return json.loads(json.dumps(data, default=json_serialize).encode())
@@ -167,3 +178,13 @@ def _get_default_figure(self) -> None:
         raw = fig.to_dict()
         raw["layout"].pop("template")  # remove boilerplate
         self.result.figure = raw
+
+    def get_template(self) -> IndicatorTemplates:
+        """Get template for indicator."""
+        indicator_key = camel_to_snake(type(self).__name__)
+        dir = get_module_dir(f"ohsome_quality_api.indicators.{indicator_key}")
+        file = os.path.join(dir, "templates.yaml")
+        with open(file, "r") as file:
+            raw = yaml.safe_load(file)
+        templates = IndicatorTemplates(**raw)
+        return templates

ohsome_quality_api/indicators/base.py

@@ -111,7 +111,7 @@ def calculate(self) -> None:
             edge_case = self.check_minor_edge_cases(key)
             # ZeroDivisionError can not occur because of `check_major_edge_cases()`
             self.ratio[key] = self.area_osm[key] / self.area_ref[key]
-            template = Template(self.metadata.result_description)
+            template = Template(self.templates.result_description)
             description = template.substitute(
                 ratio=round(self.ratio[key] * 100, 2),
                 coverage=round(self.area_cov[key] * 100, 2),
@@ -130,13 +130,13 @@ def calculate(self) -> None:
                 self.result.class_ = 3
             elif self.th_low > self.result.value >= 0:
                 self.result.class_ = 1
-            label_description = self.metadata.label_description[self.result.label]
+            label_description = self.templates.label_description[self.result.label]
             self.result.description = " ".join((label_description, result_description))
         elif major_edge_case:
-            label_description = self.metadata.label_description[self.result.label]
+            label_description = self.templates.label_description[self.result.label]
             self.result.description = " ".join((label_description, result_description))
         else:
-            label_description = self.metadata.label_description[self.result.label]
+            label_description = self.templates.label_description[self.result.label]
             edge_case = (
                 "OSM has substantivly more buildings than the reference datasets. The "
                 "reference dataset is likely to miss many buildings."

ohsome_quality_api/indicators/building_comparison/indicator.py

@@ -0,0 +1,12 @@
+---
+label_description:
+  red: >-
+    The completeness of OSM buildings in your area-of-interest is low.
+  yellow: >-
+    The completeness of OSM buildings in your area-of-interest is medium.
+  green: >-
+    The completeness of OSM buildings in your area-of-interest is high.
+  undefined: >-
+    Comparison could not be made.
+result_description: >-
+  The completeness in comparison to $dataset is $ratio%.