diff --git a/hololinked/storage/__init__.py b/hololinked/storage/__init__.py index d7589fca..da12a702 100644 --- a/hololinked/storage/__init__.py +++ b/hololinked/storage/__init__.py @@ -1,4 +1,4 @@ -from .database import ThingDB +from .database import ThingDB, MongoThingDB from .json_storage import ThingJSONStorage from ..utils import get_a_filename_from_instance @@ -9,6 +9,11 @@ def prepare_object_storage(instance, **kwargs): ): filename = kwargs.get("json_filename", f"{get_a_filename_from_instance(instance, extension='json')}") instance.db_engine = ThingJSONStorage(filename=filename, instance=instance) + elif kwargs.get( + "use_mongo_db", instance.__class__.use_mongo_db if hasattr(instance.__class__, "use_mongo_db") else False + ): + config_file = kwargs.get("db_config_file", None) + instance.db_engine = MongoThingDB(instance=instance, config_file=config_file) elif kwargs.get( "use_default_db", instance.__class__.use_default_db if hasattr(instance.__class__, "use_default_db") else False ): diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index 6186d4e6..62aa1970 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -1,12 +1,16 @@ import os import threading import typing +import base64 from sqlalchemy import create_engine, select, inspect as inspect_database from sqlalchemy.ext import asyncio as asyncio_ext from sqlalchemy.orm import sessionmaker from sqlalchemy import Integer, String, JSON, LargeBinary from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, MappedAsDataclass from sqlite3 import DatabaseError +from pymongo import MongoClient, errors as mongo_errors +from ..param import Parameterized +from ..core.property import Property from dataclasses import dataclass from ..param import Parameterized @@ -458,5 +462,124 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None: except Exception as ex: pass +class MongoThingDB: + """ + MongoDB-backed database engine for Thing properties and info. + + This class provides persistence for Thing properties using MongoDB. + Properties are stored in the 'properties' collection, with fields: + - id: Thing instance identifier + - name: property name + - serialized_value: serialized property value + + Methods mirror the interface of ThingDB for compatibility. + """ + def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None: + """ + Initialize MongoThingDB for a Thing instance. + Connects to MongoDB and sets up collections. + """ + self.thing_instance = instance + self.id = instance.id + self.config = self.load_conf(config_file) + self.client = MongoClient(self.config.get("mongo_uri", "mongodb://localhost:27017")) + self.db = self.client[self.config.get("database", "hololinked")] + self.properties = self.db["properties"] + self.things = self.db["things"] + + @classmethod + def load_conf(cls, config_file: str) -> typing.Dict[str, typing.Any]: + """ + Load configuration from JSON file if provided. + """ + if not config_file: + return {} + elif config_file.endswith(".json"): + with open(config_file, "r") as file: + return JSONSerializer.load(file) + else: + raise ValueError(f"config files of extension - ['json'] expected, given file name {config_file}") + + def fetch_own_info(self): + """ + Fetch Thing instance metadata from the 'things' collection. + """ + doc = self.things.find_one({"id": self.id}) + return doc + + def get_property(self, property: typing.Union[str, Property], deserialized: bool = True) -> typing.Any: + """ + Get a property value from MongoDB for this Thing. + If deserialized=True, returns the Python value. + """ + name = property if isinstance(property, str) else property.name + doc = self.properties.find_one({"id": self.id, "name": name}) + if not doc: + raise mongo_errors.PyMongoError(f"property {name} not found in database") + if not deserialized: + return doc + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name) + return serializer.loads(base64.b64decode(doc["serialized_value"])) + + def set_property(self, property: typing.Union[str, Property], value: typing.Any) -> None: + """ + Set a property value in MongoDB for this Thing. + Value is serialized before storage. + """ + name = property if isinstance(property, str) else property.name + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name) + serialized_value = base64.b64encode(serializer.dumps(value)).decode("utf-8") + self.properties.update_one( + {"id": self.id, "name": name}, + {"$set": {"serialized_value": serialized_value}}, + upsert=True + ) + + def get_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any], deserialized: bool = True) -> typing.Dict[str, typing.Any]: + """ + Get multiple property values from MongoDB for this Thing. + Returns a dict of property names to values. + """ + names = [obj if isinstance(obj, str) else obj.name for obj in properties.keys()] + cursor = self.properties.find({"id": self.id, "name": {"$in": names}}) + result = {} + for doc in cursor: + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, doc["name"]) + result[doc["name"]] = doc["serialized_value"] if not deserialized else serializer.loads(base64.b64decode(doc["serialized_value"])) + return result + + def set_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any]) -> None: + """ + Set multiple property values in MongoDB for this Thing. + """ + for obj, value in properties.items(): + name = obj if isinstance(obj, str) else obj.name + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name) + serialized_value = base64.b64encode(serializer.dumps(value)).decode("utf-8") + self.properties.update_one( + {"id": self.id, "name": name}, + {"$set": {"serialized_value": serialized_value}}, + upsert=True + ) + def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typing.Any]: + cursor = self.properties.find({"id": self.id}) + result = {} + for doc in cursor: + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, doc["name"]) + result[doc["name"]] = doc["serialized_value"] if not deserialized else serializer.loads(base64.b64decode(doc["serialized_value"])) + return result + + def create_missing_properties(self, properties: typing.Dict[str, Property], get_missing_property_names: bool = False) -> typing.Any: + missing_props = [] + existing_props = self.get_all_properties() + for name, new_prop in properties.items(): + if name not in existing_props: + serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, new_prop.name) + serialized_value = base64.b64encode(serializer.dumps(getattr(self.thing_instance, new_prop.name))).decode("utf-8") + self.properties.insert_one({"id": self.id, "name": new_prop.name, "serialized_value": serialized_value}) + missing_props.append(name) + if get_missing_property_names: + return missing_props + __all__ = [BaseAsyncDB.__name__, BaseSyncDB.__name__, ThingDB.__name__, batch_db_commit.__name__] diff --git a/pyproject.toml b/pyproject.toml index 0cf839b6..cc9aac93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "jsonschema>=4.22.0,<5.0", "httpx>=0.28.1,<29.0", "sniffio>=1.3.1,<2.0", + "pymongo>=4.15.2", ] [project.urls] diff --git a/tests/test_07_properties.py b/tests/test_07_properties.py index a61d2962..533d277c 100644 --- a/tests/test_07_properties.py +++ b/tests/test_07_properties.py @@ -11,10 +11,6 @@ class TestProperty(TestCase): - @classmethod - def setUpClass(self): - super().setUpClass() - print(f"test property with {self.__name__}") def test_01_simple_class_property(self): """Test basic class property functionality""" diff --git a/tests/working/test_07_properties_mongodb.py b/tests/working/test_07_properties_mongodb.py new file mode 100644 index 00000000..54def57b --- /dev/null +++ b/tests/working/test_07_properties_mongodb.py @@ -0,0 +1,80 @@ +import unittest +from hololinked.core.property import Property +from hololinked.core import Thing +from hololinked.storage.database import MongoThingDB +from pymongo import MongoClient + +class TestMongoDBOperations(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Clear MongoDB 'properties' collection before tests + try: + client = MongoClient("mongodb://localhost:27017") + db = client["hololinked"] + db["properties"].delete_many({}) + except Exception as e: + print(f"Warning: Could not clear MongoDB test data: {e}") + + def test_mongo_string_property(self): + class MongoTestThing(Thing): + str_prop = Property(default="hello", db_persist=True) + instance = MongoTestThing(id="mongo_str", use_mongo_db=True) + instance.str_prop = "world" + value_from_db = instance.db_engine.get_property("str_prop") + self.assertEqual(value_from_db, "world") + + def test_mongo_float_property(self): + class MongoTestThing(Thing): + float_prop = Property(default=1.23, db_persist=True) + instance = MongoTestThing(id="mongo_float", use_mongo_db=True) + instance.float_prop = 4.56 + value_from_db = instance.db_engine.get_property("float_prop") + self.assertAlmostEqual(value_from_db, 4.56) + + def test_mongo_bool_property(self): + class MongoTestThing(Thing): + bool_prop = Property(default=False, db_persist=True) + instance = MongoTestThing(id="mongo_bool", use_mongo_db=True) + instance.bool_prop = True + value_from_db = instance.db_engine.get_property("bool_prop") + self.assertTrue(value_from_db) + + def test_mongo_dict_property(self): + class MongoTestThing(Thing): + dict_prop = Property(default={"a": 1}, db_persist=True) + instance = MongoTestThing(id="mongo_dict", use_mongo_db=True) + instance.dict_prop = {"b": 2, "c": 3} + value_from_db = instance.db_engine.get_property("dict_prop") + self.assertEqual(value_from_db, {"b": 2, "c": 3}) + + def test_mongo_list_property(self): + class MongoTestThing(Thing): + list_prop = Property(default=[1, 2], db_persist=True) + instance = MongoTestThing(id="mongo_list", use_mongo_db=True) + instance.list_prop = [3, 4, 5] + value_from_db = instance.db_engine.get_property("list_prop") + self.assertEqual(value_from_db, [3, 4, 5]) + + def test_mongo_none_property(self): + class MongoTestThing(Thing): + none_prop = Property(default=None, db_persist=True, allow_None=True) + instance = MongoTestThing(id="mongo_none", use_mongo_db=True) + instance.none_prop = None + value_from_db = instance.db_engine.get_property("none_prop") + self.assertIsNone(value_from_db) + + def test_mongo_property_persistence(self): + thing_id = "mongo_test_persistence_unique" + prop_name = "test_prop_unique" + client = MongoClient("mongodb://localhost:27017") + db = client["hololinked"] + db["properties"].delete_many({"id": thing_id, "name": prop_name}) + class MongoTestThing(Thing): + test_prop_unique = Property(default=123, db_persist=True) + instance = MongoTestThing(id=thing_id, use_mongo_db=True) + instance.test_prop_unique = 456 + value_from_db = instance.db_engine.get_property(prop_name) + self.assertEqual(value_from_db, 456) + +if __name__ == "__main__": + unittest.main()