SelfQuery support for deeplake (#7888)

Added support SelfQuery for Deeplake

libs/langchain/langchain/retrievers/self_query/base.py

@@ -13,6 +13,7 @@ from langchain.chains.query_constructor.base import load_query_constructor_chain
 from langchain.chains.query_constructor.ir import StructuredQuery, Visitor
 from langchain.chains.query_constructor.schema import AttributeInfo
 from langchain.retrievers.self_query.chroma import ChromaTranslator
+from langchain.retrievers.self_query.deeplake import DeepLakeTranslator
 from langchain.retrievers.self_query.myscale import MyScaleTranslator
 from langchain.retrievers.self_query.pinecone import PineconeTranslator
 from langchain.retrievers.self_query.qdrant import QdrantTranslator
@@ -21,6 +22,7 @@ from langchain.schema import BaseRetriever, Document
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.vectorstores import (
     Chroma,
+    DeepLake,
     MyScale,
     Pinecone,
     Qdrant,
@@ -38,6 +40,7 @@ def _get_builtin_translator(vectorstore: VectorStore) -> Visitor:
         Weaviate: WeaviateTranslator,
         Qdrant: QdrantTranslator,
         MyScale: MyScaleTranslator,
+        DeepLake: DeepLakeTranslator,
     }
     if vectorstore_cls not in BUILTIN_TRANSLATORS:
         raise ValueError(

libs/langchain/langchain/retrievers/self_query/deeplake.py

@@ -0,0 +1,86 @@
+"""Logic for converting internal query language to a valid Chroma query."""
+from typing import Tuple, Union
+
+from langchain.chains.query_constructor.ir import (
+    Comparator,
+    Comparison,
+    Operation,
+    Operator,
+    StructuredQuery,
+    Visitor,
+)
+
+COMPARATOR_TO_TQL = {
+    Comparator.EQ: "==",
+    Comparator.GT: ">",
+    Comparator.GTE: ">=",
+    Comparator.LT: "<",
+    Comparator.LTE: "<=",
+}
+
+
+OPERATOR_TO_TQL = {
+    Operator.AND: "and",
+    Operator.OR: "or",
+}
+
+
+def can_cast_to_float(string: str) -> bool:
+    try:
+        float(string)
+        return True
+    except ValueError:
+        return False
+
+
+class DeepLakeTranslator(Visitor):
+    """Logic for converting internal query language elements to valid filters."""
+
+    allowed_operators = [Operator.AND, Operator.OR]
+    """Subset of allowed logical operators."""
+    allowed_comparators = [
+        Comparator.EQ,
+        Comparator.GT,
+        Comparator.GTE,
+        Comparator.LT,
+        Comparator.LTE,
+    ]
+    """Subset of allowed logical comparators."""
+
+    def _format_func(self, func: Union[Operator, Comparator]) -> str:
+        self._validate_func(func)
+        if isinstance(func, Operator):
+            value = OPERATOR_TO_TQL[func.value]  # type: ignore
+        elif isinstance(func, Comparator):
+            value = COMPARATOR_TO_TQL[func.value]  # type: ignore
+        return f"{value}"
+
+    def visit_operation(self, operation: Operation) -> str:
+        args = [arg.accept(self) for arg in operation.arguments]
+        operator = self._format_func(operation.operator)
+        return "(" + (" " + operator + " ").join(args) + ")"
+
+    def visit_comparison(self, comparison: Comparison) -> str:
+        comparator = self._format_func(comparison.comparator)
+        values = comparison.value
+        if isinstance(values, list):
+            tql = []
+            for value in values:
+                comparison.value = value
+                tql.append(self.visit_comparison(comparison))
+
+            return "(" + (" or ").join(tql) + ")"
+
+        if not can_cast_to_float(comparison.value):
+            values = f"'{values}'"
+        return f"metadata['{comparison.attribute}'] {comparator} {values}"
+
+    def visit_structured_query(
+        self, structured_query: StructuredQuery
+    ) -> Tuple[str, dict]:
+        if structured_query.filter is None:
+            kwargs = {}
+        else:
+            tqL = f"SELECT * WHERE {structured_query.filter.accept(self)}"
+            kwargs = {"tql": tqL}
+        return structured_query.query, kwargs

libs/langchain/langchain/vectorstores/deeplake.py

@@ -219,14 +219,14 @@ class DeepLake(VectorStore):
 
     def _search_tql(
         self,
-        tql_query: Optional[str],
+        tql: Optional[str],
         exec_option: Optional[str] = None,
         **kwargs: Any,
     ) -> List[Document]:
         """Function for performing tql_search.
 
         Args:
-            tql_query (str): TQL Query string for direct evaluation.
+            tql (str): TQL Query string for direct evaluation.
                 Available only for `compute_engine` and `tensor_db`.
             exec_option (str, optional): Supports 3 ways to search.
                 Could be "python", "compute_engine" or "tensor_db". Default is "python".
@@ -249,7 +249,7 @@ class DeepLake(VectorStore):
             ValueError: If return_score is True but some condition is not met.
         """
         result = self.vectorstore.search(
-            query=tql_query,
+            query=tql,
             exec_option=exec_option,
         )
         metadatas = result["metadata"]
@@ -328,9 +328,9 @@ class DeepLake(VectorStore):
             ValueError: if both `embedding` and `embedding_function` are not specified.
         """
 
-        if kwargs.get("tql_query"):
+        if kwargs.get("tql"):
             return self._search_tql(
-                tql_query=kwargs["tql_query"],
+                tql=kwargs["tql"],
                 exec_option=exec_option,
                 return_score=return_score,
                 embedding=embedding,
@@ -423,7 +423,7 @@ class DeepLake(VectorStore):
             >>> # Run tql search:
             >>> data = vector_store.similarity_search(
             ...     query=None,
-            ...     tql_query="SELECT * WHERE id == <id>",
+            ...     tql="SELECT * WHERE id == <id>",
             ...     exec_option="compute_engine",
             ... )
 

libs/langchain/tests/unit_tests/retrievers/self_query/test_deeplake.py

@@ -0,0 +1,33 @@
+from langchain.chains.query_constructor.ir import (
+    Comparator,
+    Comparison,
+    Operation,
+    Operator,
+)
+from langchain.retrievers.self_query.deeplake import DeepLakeTranslator
+
+DEFAULT_TRANSLATOR = DeepLakeTranslator()
+
+
+def test_visit_comparison() -> None:
+    comp = Comparison(comparator=Comparator.LT, attribute="foo", value=["1", "2"])
+    expected = "(metadata['foo'] < 1 or metadata['foo'] < 2)"
+    actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
+    assert expected == actual
+
+
+def test_visit_operation() -> None:
+    op = Operation(
+        operator=Operator.AND,
+        arguments=[
+            Comparison(comparator=Comparator.LT, attribute="foo", value=2),
+            Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
+            Comparison(comparator=Comparator.LT, attribute="abc", value=["1", "2"]),
+        ],
+    )
+    expected = (
+        "(metadata['foo'] < 2 and metadata['bar'] == 'baz' "
+        "and (metadata['abc'] < 1 or metadata['abc'] < 2))"
+    )
+    actual = DEFAULT_TRANSLATOR.visit_operation(op)
+    assert expected == actual