Adding Self-querying for Vectara (#10332)

- Description: Adding support for self-querying to Vectara integration
  - Issue: per customer request
  - Tag maintainer: @rlancemartin @baskaryan 
  - Twitter handle: @ofermend 

Also updated some documentation, added self-query testing, and a demo
notebook with self-query example.

libs/langchain/langchain/retrievers/self_query/base.py

@@ -16,6 +16,7 @@ from langchain.retrievers.self_query.milvus import MilvusTranslator
 from langchain.retrievers.self_query.myscale import MyScaleTranslator
 from langchain.retrievers.self_query.pinecone import PineconeTranslator
 from langchain.retrievers.self_query.qdrant import QdrantTranslator
+from langchain.retrievers.self_query.vectara import VectaraTranslator
 from langchain.retrievers.self_query.weaviate import WeaviateTranslator
 from langchain.schema import BaseRetriever, Document
 from langchain.schema.language_model import BaseLanguageModel
@@ -28,6 +29,7 @@ from langchain.vectorstores import (
     MyScale,
     Pinecone,
     Qdrant,
+    Vectara,
     VectorStore,
     Weaviate,
 )
@@ -41,6 +43,7 @@ def _get_builtin_translator(vectorstore: VectorStore) -> Visitor:
         Chroma: ChromaTranslator,
         DashVector: DashvectorTranslator,
         Weaviate: WeaviateTranslator,
+        Vectara: VectaraTranslator,
         Qdrant: QdrantTranslator,
         MyScale: MyScaleTranslator,
         DeepLake: DeepLakeTranslator,

libs/langchain/langchain/retrievers/self_query/vectara.py

@@ -0,0 +1,69 @@
+from typing import Tuple, Union
+
+from langchain.chains.query_constructor.ir import (
+    Comparator,
+    Comparison,
+    Operation,
+    Operator,
+    StructuredQuery,
+    Visitor,
+)
+
+
+def process_value(value: Union[int, float, str]) -> str:
+    if isinstance(value, str):
+        return f"'{value}'"
+    else:
+        return str(value)
+
+
+class VectaraTranslator(Visitor):
+    """Translate `Vectara` internal query language elements to valid filters."""
+
+    allowed_operators = [Operator.AND, Operator.OR]
+    """Subset of allowed logical operators."""
+    allowed_comparators = [
+        Comparator.EQ,
+        Comparator.NE,
+        Comparator.GT,
+        Comparator.GTE,
+        Comparator.LT,
+        Comparator.LTE,
+    ]
+    """Subset of allowed logical comparators."""
+
+    def _format_func(self, func: Union[Operator, Comparator]) -> str:
+        map_dict = {
+            Operator.AND: " and ",
+            Operator.OR: " or ",
+            Comparator.EQ: "=",
+            Comparator.NE: "!=",
+            Comparator.GT: ">",
+            Comparator.GTE: ">=",
+            Comparator.LT: "<",
+            Comparator.LTE: "<=",
+        }
+        self._validate_func(func)
+        return map_dict[func]
+
+    def visit_operation(self, operation: Operation) -> str:
+        args = [arg.accept(self) for arg in operation.arguments]
+        operator = self._format_func(operation.operator)
+        return "( " + operator.join(args) + " )"
+
+    def visit_comparison(self, comparison: Comparison) -> str:
+        comparator = self._format_func(comparison.comparator)
+        processed_value = process_value(comparison.value)
+        attribute = comparison.attribute
+        return (
+            "( " + "doc." + attribute + " " + comparator + " " + processed_value + " )"
+        )
+
+    def visit_structured_query(
+        self, structured_query: StructuredQuery
+    ) -> Tuple[str, dict]:
+        if structured_query.filter is None:
+            kwargs = {}
+        else:
+            kwargs = {"filter": structured_query.filter.accept(self)}
+        return structured_query.query, kwargs

libs/langchain/langchain/vectorstores/vectara.py

@@ -396,8 +396,12 @@ class Vectara(VectorStore):
                     vectara_api_key=api_key,
                 )
         """
-        # Note: Vectara generates its own embeddings, so we ignore the provided
-        # embeddings (required by interface)
+        # Notes:
+        # * Vectara generates its own embeddings, so we ignore the provided
+        #   embeddings (required by interface)
+        # * when metadatas[] are provided they are associated with each "part"
+        #   in Vectara. doc_metadata can be used to provide additional metadata
+        #   for the document itself (applies to all "texts" in this call)
         doc_metadata = kwargs.pop("doc_metadata", {})
         vectara = cls(**kwargs)
         vectara.add_texts(texts, metadatas, doc_metadata=doc_metadata, **kwargs)

libs/langchain/tests/integration_tests/document_loaders/test_polars_dataframe.py

@@ -34,8 +34,6 @@ def test_load_returns_list_of_documents(sample_data_frame: pl.DataFrame) -> None
 def test_load_converts_dataframe_columns_to_document_metadata(
     sample_data_frame: pl.DataFrame,
 ) -> None:
-    import polars as pl
-
     loader = PolarsDataFrameLoader(sample_data_frame)
     docs = loader.load()
 

libs/langchain/tests/unit_tests/retrievers/self_query/test_vectara.py

@@ -0,0 +1,71 @@
+from typing import Dict, Tuple
+
+from langchain.chains.query_constructor.ir import (
+    Comparator,
+    Comparison,
+    Operation,
+    Operator,
+    StructuredQuery,
+)
+from langchain.retrievers.self_query.vectara import VectaraTranslator
+
+DEFAULT_TRANSLATOR = VectaraTranslator()
+
+
+def test_visit_comparison() -> None:
+    comp = Comparison(comparator=Comparator.LT, attribute="foo", value="1")
+    expected = "( doc.foo < '1' )"
+    actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
+    assert expected == actual
+
+
+def test_visit_operation() -> None:
+    op = Operation(
+        operator=Operator.AND,
+        arguments=[
+            Comparison(comparator=Comparator.LT, attribute="foo", value=2),
+            Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
+            Comparison(comparator=Comparator.LT, attribute="abc", value=1),
+        ],
+    )
+    expected = "( ( doc.foo < 2 ) and ( doc.bar = 'baz' ) and ( doc.abc < 1 ) )"
+    actual = DEFAULT_TRANSLATOR.visit_operation(op)
+    assert expected == actual
+
+
+def test_visit_structured_query() -> None:
+    query = "What is the capital of France?"
+    structured_query = StructuredQuery(
+        query=query,
+        filter=None,
+        limit=None,
+    )
+    expected: Tuple[str, Dict] = (query, {})
+    actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
+    assert expected == actual
+
+    comp = Comparison(comparator=Comparator.LT, attribute="foo", value=1)
+    expected = (query, {"filter": "( doc.foo < 1 )"})
+    structured_query = StructuredQuery(
+        query=query,
+        filter=comp,
+        limit=None,
+    )
+    actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
+    assert expected == actual
+
+    op = Operation(
+        operator=Operator.AND,
+        arguments=[
+            Comparison(comparator=Comparator.LT, attribute="foo", value=2),
+            Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
+            Comparison(comparator=Comparator.LT, attribute="abc", value=1),
+        ],
+    )
+    structured_query = StructuredQuery(query=query, filter=op, limit=None)
+    expected = (
+        query,
+        {"filter": "( ( doc.foo < 2 ) and ( doc.bar = 'baz' ) and ( doc.abc < 1 ) )"},
+    )
+    actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
+    assert expected == actual