Support client-side schema validation using Pydantic (#304)

This PR implements a layered architecture for managing and validating
searchable data in Redis, with clear separation of concerns between
schema definition, data validation, and storage operations.

- `IndexSchema` provides the blueprint for data structure and
constraints
- Defines fields with specific types (TEXT, TAG, NUMERIC, GEO, VECTOR)
- Supports different storage types (HASH, JSON) with appropriate
configuration

- `SchemaModelGenerator` dynamically creates Pydantic models from schema
definitions
- Implements a caching mechanism to avoid redundant model generation
- Maps Redis field types to appropriate Python/Pydantic types
- Provides type-specific validators:
- VECTOR: validates dimensions and value ranges (e.g., INT8 range
checks)
    - GEO: validates geographic coordinate format
    - NUMERIC: prevents boolean values

- `BaseStorage` is the abstract class provides the foundation for Redis
operations
- Specialized implementations (HashStorage, JsonStorage) for different
Redis data types
- Enforces schema validation during write operations when set to True
- Implements optimized batch operations using Redis pipelines
- Supports both synchronous and asynchronous interfaces
- Handles key generation, preprocessing, and error handling

The `SearchIndex` contains the setting `validate_on_load`, which
defaults on `False`.

Objects are preprocessed and validated against the schema
Objects are prepared with appropriate keys
Batch writing occurs using Redis pipelines for efficiency
TTL (expiration) can be applied if specified

Keys are fetched in batches using pipelines
Data is converted from Redis format to Python objects
Bytes are automatically converted to appropriate types

Author: tylerhutcherson

.github/workflows/test.yml

@@ -133,6 +133,11 @@ jobs:
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
 
+      - name: Set HuggingFace token
+        run: |
+          mkdir -p ~/.huggingface
+          echo '{"token":"${{ secrets.HF_TOKEN }}"}' > ~/.huggingface/token 
+
       - name: Run tests
         if: matrix.connection == 'plain' && matrix.redis-version == 'latest'
         env:
@@ -149,6 +154,7 @@ jobs:
           OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
           make test-all
 
@@ -173,6 +179,7 @@ jobs:
           OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: |
           docker run -d --name redis -p 6379:6379 redis/redis-stack-server:latest
           if [[ "${{ matrix.python-version }}" > "3.9" ]]; then

docs/user_guide/01_getting_started.ipynb

@@ -34,6 +34,7 @@ tabulate = "^0.9.0"
 ml-dtypes = "^0.4.0"
 python-ulid = "^3.0.0"
 nltk = { version = "^3.8.1", optional = true }
+jsonpath-ng = "^1.5.0"
 openai = { version = "^1.13.0", optional = true }
 sentence-transformers = { version = "^3.4.0", optional = true }
 scipy = [

poetry.lock

@@ -1,10 +1,32 @@
-class RedisVLException(Exception):
-    """Base RedisVL exception"""
+"""
+RedisVL Exception Classes
 
+This module defines all custom exceptions used throughout the RedisVL library.
+"""
 
-class RedisModuleVersionError(RedisVLException):
-    """Invalid module versions installed"""
 
+class RedisVLError(Exception):
+    """Base exception for all RedisVL errors."""
 
-class RedisSearchError(RedisVLException):
-    """Error while performing a search or aggregate request"""
+    pass
+
+
+class RedisModuleVersionError(RedisVLError):
+    """Error raised when required Redis modules are missing or have incompatible versions."""
+
+    pass
+
+
+class RedisSearchError(RedisVLError):
+    """Error raised for Redis Search specific operations."""
+
+    pass
+
+
+class SchemaValidationError(RedisVLError):
+    """Error when validating data against a schema."""
+
+    def __init__(self, message, index=None):
+        if index is not None:
+            message = f"Validation failed for object at index {index}: {message}"
+        super().__init__(message)

pyproject.toml

@@ -95,12 +95,8 @@ def __init__(
         }
 
         # Use the index name as the key prefix by default
-        if "prefix" in kwargs:
-            prefix = kwargs["prefix"]
-        else:
-            prefix = name
-
-        dtype = kwargs.get("dtype")
+        prefix = kwargs.pop("prefix", name)
+        dtype = kwargs.pop("dtype", None)
 
         # Validate a provided vectorizer or set the default
         if vectorizer:
@@ -111,7 +107,10 @@ def __init__(
                     f"Provided dtype {dtype} does not match vectorizer dtype {vectorizer.dtype}"
                 )
         else:
-            vectorizer_kwargs = {"dtype": dtype} if dtype else {}
+            vectorizer_kwargs = kwargs
+
+            if dtype:
+                vectorizer_kwargs.update(**{"dtype": dtype})
 
             vectorizer = HFTextVectorizer(
                 model="sentence-transformers/all-mpnet-base-v2",

redisvl/exceptions.py

@@ -72,7 +72,7 @@ def __init__(
             connection_kwargs (Dict[str, Any]): The connection arguments
                 for the redis client. Defaults to empty {}.
         """
-        dtype = kwargs.get("dtype")
+        dtype = kwargs.pop("dtype", None)
 
         # Validate a provided vectorizer or set the default
         if vectorizer:
@@ -83,8 +83,15 @@ def __init__(
                     f"Provided dtype {dtype} does not match vectorizer dtype {vectorizer.dtype}"
                 )
         else:
-            vectorizer_kwargs = {"dtype": dtype} if dtype else {}
-            vectorizer = HFTextVectorizer(**vectorizer_kwargs)
+            vectorizer_kwargs = kwargs
+
+            if dtype:
+                vectorizer_kwargs.update(**{"dtype": dtype})
+
+            vectorizer = HFTextVectorizer(
+                model="sentence-transformers/all-mpnet-base-v2",
+                **vectorizer_kwargs,
+            )
 
         if routing_config is None:
             routing_config = RoutingConfig()

redisvl/extensions/llmcache/semantic.py

@@ -71,7 +71,7 @@ def __init__(
         super().__init__(name, session_tag)
 
         prefix = prefix or name
-        dtype = kwargs.get("dtype")
+        dtype = kwargs.pop("dtype", None)
 
         # Validate a provided vectorizer or set the default
         if vectorizer:
@@ -82,10 +82,13 @@ def __init__(
                     f"Provided dtype {dtype} does not match vectorizer dtype {vectorizer.dtype}"
                 )
         else:
-            vectorizer_kwargs = {"dtype": dtype} if dtype else {}
+            vectorizer_kwargs = kwargs
+
+            if dtype:
+                vectorizer_kwargs.update(**{"dtype": dtype})
 
             vectorizer = HFTextVectorizer(
-                model="sentence-transformers/msmarco-distilbert-cos-v5",
+                model="sentence-transformers/all-mpnet-base-v2",
                 **vectorizer_kwargs,
             )