wip: debugging aggregations and filters

Author: justin-cechmanek

redisvl/query/aggregate.py

@@ -32,26 +32,26 @@ class HybridAggregationQuery(AggregationQuery):
     def __init__(
         self,
         text: str,
-        text_field: str,
+        text_field_name: str,
         vector: Union[bytes, List[float]],
-        vector_field: str,
+        vector_field_name: str,
         text_scorer: str = "BM25STD",
         filter_expression: Optional[Union[str, FilterExpression]] = None,
         alpha: float = 0.7,
         dtype: str = "float32",
         num_results: int = 10,
         return_fields: Optional[List[str]] = None,
         stopwords: Optional[Union[str, Set[str]]] = "english",
-        dialect: int = 4,
+        dialect: int = 2,
     ):
         """
         Instantiages a HybridAggregationQuery object.
 
         Args:
             text (str): The text to search for.
-            text_field (str): The text field name to search in.
+            text_field_name (str): The text field name to search in.
             vector (Union[bytes, List[float]]): The vector to perform vector similarity search.
-            vector_field (str): The vector field name to search in.
+            vector_field_name (str): The vector field name to search in.
             text_scorer (str, optional): The text scorer to use. Options are {TFIDF, TFIDF.DOCNORM,
                 BM25, DISMAX, DOCSCORE, BM25STD}. Defaults to "BM25STD".
             filter_expression (Optional[FilterExpression], optional): The filter expression to use.
@@ -67,7 +67,7 @@ def __init__(
                 provided then a default set of stopwords for that language will be used. if a list,
                 set, or tuple of strings is provided then those will be used as stopwords.
                 Defaults to "english". if set to "None" then no stopwords will be removed.
-            dialect (int, optional): The Redis dialect version. Defaults to 4.
+            dialect (int, optional): The Redis dialect version. Defaults to 2.
 
         Raises:
             ValueError: If the text string is empty, or if the text string becomes empty after
@@ -82,9 +82,9 @@ def __init__(
 
             query = HybridAggregationQuery(
                 text="example text",
-                text_field="text_field",
+                text_field_name="text_field",
                 vector=[0.1, 0.2, 0.3],
-                vector_field="vector_field",
+                vector_field_name="vector_field",
                 text_scorer="BM25STD",
                 filter_expression=None,
                 alpha=0.7,
@@ -102,16 +102,19 @@ def __init__(
             raise ValueError("text string cannot be empty")
 
         self._text = text
-        self._text_field = text_field
+        self._text_field = text_field_name
         self._vector = vector
-        self._vector_field = vector_field
+        self._vector_field = vector_field_name
         self._filter_expression = filter_expression
         self._alpha = alpha
         self._dtype = dtype
         self._num_results = num_results
         self.set_stopwords(stopwords)
 
         query_string = self._build_query_string()
+        ####
+        print('query_string: ', query_string)
+        ####
         super().__init__(query_string)
 
         self.scorer(text_scorer)  # type: ignore[attr-defined]

redisvl/query/query.py

@@ -691,7 +691,7 @@ class TextQuery(FilterQuery):
     def __init__(
         self,
         text: str,
-        text_field: str,
+        text_field_name: str,
         text_scorer: str = "BM25STD",
         filter_expression: Optional[Union[str, FilterExpression]] = None,
         return_fields: Optional[List[str]] = None,
@@ -708,7 +708,7 @@ def __init__(
 
         Args:
             text (str): The text string to perform the text search with.
-            text_field (str): The name of the document field to perform text search on.
+            text_field_name (str): The name of the document field to perform text search on.
             text_scorer (str, optional): The text scoring algorithm to use.
                 Defaults to BM25STD. Options are {TFIDF, BM25STD, BM25, DOCNORM, DISMAX, DOCSCORE}.
                 See https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/scoring/
@@ -740,7 +740,7 @@ def __init__(
             TypeError: If stopwords is not a valid iterable set of strings.
         """
         self._text = text
-        self._text_field = text_field
+        self._text_field = text_field_name
         self._text_scorer = text_scorer
         self._num_results = num_results
 

tests/integration/test_aggregation.py

@@ -1,9 +1,9 @@
-from datetime import timedelta
-
 import pytest
 from redis.commands.search.aggregation import AggregateResult
 from redis.commands.search.result import Result
 
+from redisvl.redis.connection import compare_versions
+
 from redisvl.index import SearchIndex
 from redisvl.query import HybridAggregationQuery
 from redisvl.query.filter import (
@@ -13,31 +13,12 @@
     Num,
     Tag,
     Text,
-    Timestamp,
 )
 from redisvl.redis.utils import array_to_buffer
 
-# TODO expand to multiple schema types and sync + async
-
-vector = ([0.1, 0.1, 0.5],)
-vector_field_name = ("user_embedding",)
-return_fields = (
-    [
-        "user",
-        "credit_score",
-        "age",
-        "job",
-        "location",
-        "last_updated",
-    ],
-)
-filter_expression = (Tag("credit_score") == "high",)
-distance_threshold = (0.2,)
-
 
 @pytest.fixture
 def index(sample_data, redis_url):
-    # construct a search index from the schema
     index = SearchIndex.from_dict(
         {
             "index": {
@@ -70,7 +51,7 @@ def index(sample_data, redis_url):
     # create the index (no data yet)
     index.create(overwrite=True)
 
-    # Prepare and load the data
+    # prepare and load the data
     def hash_preprocess(item: dict) -> dict:
         return {
             **item,
@@ -87,7 +68,10 @@ def hash_preprocess(item: dict) -> dict:
 
 
 def test_aggregation_query(index):
-    # *=>[KNN 7 @user_embedding $vector AS vector_distance]
+    redis_version = index.client.info()["redis_version"]
+    if not compare_versions(redis_version, "7.2.0"):
+        pytest.skip("Not using a late enough version of Redis")
+
     text = "a medical professional with expertise in lung cancer"
     text_field = "description"
     vector = [0.1, 0.1, 0.5]
@@ -96,17 +80,16 @@ def test_aggregation_query(index):
 
     hybrid_query = HybridAggregationQuery(
         text=text,
-        text_field=text_field,
+        text_field_name=text_field,
         vector=vector,
-        vector_field=vector_field,
+        vector_field_name=vector_field,
         return_fields=return_fields,
     )
 
     results = index.aggregate_query(hybrid_query)
     assert isinstance(results, list)
     assert len(results) == 7
     for doc in results:
-        # ensure all return fields present
         assert doc["user"] in [
             "john",
             "derrick",
@@ -121,12 +104,11 @@ def test_aggregation_query(index):
         assert doc["job"] in ["engineer", "doctor", "dermatologist", "CEO", "dentist"]
         assert doc["credit_score"] in ["high", "low", "medium"]
 
-    # test num_results
     hybrid_query = HybridAggregationQuery(
         text=text,
-        text_field=text_field,
+        text_field_name=text_field,
         vector=vector,
-        vector_field=vector_field,
+        vector_field_name=vector_field,
         num_results=3,
     )
 
@@ -139,7 +121,7 @@ def test_aggregation_query(index):
     )
 
 
-def test_empty_query_string(index):
+def test_empty_query_string():
     text = ""
     text_field = "description"
     vector = [0.1, 0.1, 0.5]
@@ -149,30 +131,86 @@ def test_empty_query_string(index):
     # test if text is empty
     with pytest.raises(ValueError):
         hybrid_query = HybridAggregationQuery(
-            text=text, text_field=text_field, vector=vector, vector_field=vector_field
+            text=text, text_field_name=text_field, vector=vector, vector_field_name=vector_field
         )
 
     # test if text becomes empty after stopwords are removed
     text = "with a for but and"  # will all be removed as default stopwords
     with pytest.raises(ValueError):
         hybrid_query = HybridAggregationQuery(
-            text=text, text_field=text_field, vector=vector, vector_field=vector_field
+            text=text, text_field_name=text_field, vector=vector, vector_field_name=vector_field
         )
 
+def test_aggregation_query_filter(index):
+    redis_version = index.client.info()["redis_version"]
+    if not compare_versions(redis_version, "7.2.0"):
+        pytest.skip("Not using a late enough version of Redis")
 
-def test_aggregate_query_stopwords(index):
     text = "a medical professional with expertise in lung cancer"
     text_field = "description"
     vector = [0.1, 0.1, 0.5]
     vector_field = "user_embedding"
     return_fields = ["user", "credit_score", "age", "job", "location", "description"]
-    return
-    # test num_results
+    filter_expression = (Tag("credit_score") == ("high")) & (Num("age") > 30)
+
     hybrid_query = HybridAggregationQuery(
         text=text,
-        text_field=text_field,
+        text_field_name=text_field,
         vector=vector,
-        vector_field=vector_field,
+        vector_field_name=vector_field,
+        filter_expression=filter_expression,
+        return_fields=return_fields,
+    )
+
+    results = index.aggregate_query(hybrid_query)
+    assert len(results) == 3
+    for result in results:
+        assert result["credit_score"] == "high"
+        assert int(result["age"]) > 30
+
+
+def test_aggregation_query_with_geo_filter(index):
+    redis_version = index.client.info()["redis_version"]
+    if not compare_versions(redis_version, "7.2.0"):
+        pytest.skip("Not using a late enough version of Redis")
+
+    text = "a medical professional with expertise in lung cancer"
+    text_field = "description"
+    vector = [0.1, 0.1, 0.5]
+    vector_field = "user_embedding"
+    return_fields = ["user", "credit_score", "age", "job", "location", "description"]
+    filter_expression = Geo("location") == GeoRadius(37.7749, -122.4194, 1000)
+
+    hybrid_query = HybridAggregationQuery(
+        text=text,
+        text_field_name=text_field,
+        vector=vector,
+        vector_field_name=vector_field,
+        filter_expression=filter_expression,
+        return_fields=return_fields,
+    )
+
+    results = index.aggregate_query(hybrid_query)
+    assert len(results) == 3
+    for result in results:
+        assert result["location"] is not None
+
+
+def test_aggregate_query_stopwords(index):
+    redis_version = index.client.info()["redis_version"]
+    if not compare_versions(redis_version, "7.2.0"):
+        pytest.skip("Not using a late enough version of Redis")
+
+    text = "a medical professional with expertise in lung cancer"
+    text_field = "description"
+    vector = [0.1, 0.1, 0.5]
+    vector_field = "user_embedding"
+
+    hybrid_query = HybridAggregationQuery(
+        text=text,
+        text_field_name=text_field,
+        vector=vector,
+        vector_field_name=vector_field,
         alpha=0.5,
         stopwords=["medical", "expertise"],
     )
@@ -182,3 +220,30 @@ def test_aggregate_query_stopwords(index):
     for r in results:
         assert r["text_score"] == 0
         assert r["hybrid_score"] == 0.5 * r["vector_similarity"]
+
+
+def test_aggregate_query_text_filter(index):
+    redis_version = index.client.info()["redis_version"]
+    if not compare_versions(redis_version, "7.2.0"):
+        pytest.skip("Not using a late enough version of Redis")
+
+    text = "a medical professional with expertise in lung cancer"
+    text_field = "description"
+    vector = [0.1, 0.1, 0.5]
+    vector_field = "user_embedding"
+    filter_expression = (Text("description") == ("medical")) | (Text("job") % ("doct*"))
+
+    hybrid_query = HybridAggregationQuery(
+        text=text,
+        text_field_name=text_field,
+        vector=vector,
+        vector_field_name=vector_field,
+        alpha=0.5,
+        filter_expression=filter_expression
+        )
+
+    results = index.aggregate_query(hybrid_query)
+    assert len(results) == 7
+    for result in results:
+        assert result["text_score"] == 0
+        assert result["hybrid_score"] == 0.5 * result["vector_similarity"]