core[minor], langchain[patch], community[patch]: mv StructuredQuery (#20849)

mv StructuredQuery to core
This commit is contained in:
Bagatur 2024-04-25 09:40:26 -07:00 committed by GitHub
parent 540f384197
commit 5b83130855
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 231 additions and 198 deletions

View File

@ -110,11 +110,11 @@ def translate_filter(
lc_filter: str, allowed_fields: Optional[Sequence[str]] = None
) -> str:
from langchain.chains.query_constructor.base import fix_filter_directive
from langchain.chains.query_constructor.ir import FilterDirective
from langchain.chains.query_constructor.parser import get_parser
from langchain.retrievers.self_query.tencentvectordb import (
TencentVectorDBTranslator,
)
from langchain_core.structured_query import FilterDirective
tvdb_visitor = TencentVectorDBTranslator(allowed_fields)
flt = cast(

View File

@ -0,0 +1,143 @@
"""Internal representation of a structured query language."""
from __future__ import annotations
from abc import ABC, abstractmethod
from enum import Enum
from typing import Any, List, Optional, Sequence, Union
from langchain_core.pydantic_v1 import BaseModel
class Visitor(ABC):
"""Defines interface for IR translation using visitor pattern."""
allowed_comparators: Optional[Sequence[Comparator]] = None
allowed_operators: Optional[Sequence[Operator]] = None
def _validate_func(self, func: Union[Operator, Comparator]) -> None:
if isinstance(func, Operator) and self.allowed_operators is not None:
if func not in self.allowed_operators:
raise ValueError(
f"Received disallowed operator {func}. Allowed "
f"comparators are {self.allowed_operators}"
)
if isinstance(func, Comparator) and self.allowed_comparators is not None:
if func not in self.allowed_comparators:
raise ValueError(
f"Received disallowed comparator {func}. Allowed "
f"comparators are {self.allowed_comparators}"
)
@abstractmethod
def visit_operation(self, operation: Operation) -> Any:
"""Translate an Operation."""
@abstractmethod
def visit_comparison(self, comparison: Comparison) -> Any:
"""Translate a Comparison."""
@abstractmethod
def visit_structured_query(self, structured_query: StructuredQuery) -> Any:
"""Translate a StructuredQuery."""
def _to_snake_case(name: str) -> str:
"""Convert a name into snake_case."""
snake_case = ""
for i, char in enumerate(name):
if char.isupper() and i != 0:
snake_case += "_" + char.lower()
else:
snake_case += char.lower()
return snake_case
class Expr(BaseModel):
"""Base class for all expressions."""
def accept(self, visitor: Visitor) -> Any:
"""Accept a visitor.
Args:
visitor: visitor to accept
Returns:
result of visiting
"""
return getattr(visitor, f"visit_{_to_snake_case(self.__class__.__name__)}")(
self
)
class Operator(str, Enum):
"""Enumerator of the operations."""
AND = "and"
OR = "or"
NOT = "not"
class Comparator(str, Enum):
"""Enumerator of the comparison operators."""
EQ = "eq"
NE = "ne"
GT = "gt"
GTE = "gte"
LT = "lt"
LTE = "lte"
CONTAIN = "contain"
LIKE = "like"
IN = "in"
NIN = "nin"
class FilterDirective(Expr, ABC):
"""A filtering expression."""
class Comparison(FilterDirective):
"""A comparison to a value."""
comparator: Comparator
attribute: str
value: Any
def __init__(
self, comparator: Comparator, attribute: str, value: Any, **kwargs: Any
) -> None:
super().__init__(
comparator=comparator, attribute=attribute, value=value, **kwargs
)
class Operation(FilterDirective):
"""A logical operation over other directives."""
operator: Operator
arguments: List[FilterDirective]
def __init__(
self, operator: Operator, arguments: List[FilterDirective], **kwargs: Any
):
super().__init__(operator=operator, arguments=arguments, **kwargs)
class StructuredQuery(Expr):
"""A structured query."""
query: str
"""Query string."""
filter: Optional[FilterDirective]
"""Filtering expression."""
limit: Optional[int]
"""Limit on the number of results."""
def __init__(
self,
query: str,
filter: Optional[FilterDirective],
limit: Optional[int] = None,
**kwargs: Any,
):
super().__init__(query=query, filter=filter, limit=limit, **kwargs)

View File

@ -11,9 +11,7 @@ from langchain_core.output_parsers.json import parse_and_check_json_markdown
from langchain_core.prompts import BasePromptTemplate
from langchain_core.prompts.few_shot import FewShotPromptTemplate
from langchain_core.runnables import Runnable
from langchain.chains.llm import LLMChain
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
FilterDirective,
@ -21,6 +19,8 @@ from langchain.chains.query_constructor.ir import (
Operator,
StructuredQuery,
)
from langchain.chains.llm import LLMChain
from langchain.chains.query_constructor.parser import get_parser
from langchain.chains.query_constructor.prompt import (
DEFAULT_EXAMPLES,

View File

@ -1,143 +1,22 @@
"""Internal representation of a structured query language."""
from __future__ import annotations
from langchain_core.structured_query import (
Comparator,
Comparison,
Expr,
FilterDirective,
Operation,
Operator,
StructuredQuery,
Visitor,
)
from abc import ABC, abstractmethod
from enum import Enum
from typing import Any, List, Optional, Sequence, Union
from langchain_core.pydantic_v1 import BaseModel
class Visitor(ABC):
"""Defines interface for IR translation using visitor pattern."""
allowed_comparators: Optional[Sequence[Comparator]] = None
allowed_operators: Optional[Sequence[Operator]] = None
def _validate_func(self, func: Union[Operator, Comparator]) -> None:
if isinstance(func, Operator) and self.allowed_operators is not None:
if func not in self.allowed_operators:
raise ValueError(
f"Received disallowed operator {func}. Allowed "
f"comparators are {self.allowed_operators}"
)
if isinstance(func, Comparator) and self.allowed_comparators is not None:
if func not in self.allowed_comparators:
raise ValueError(
f"Received disallowed comparator {func}. Allowed "
f"comparators are {self.allowed_comparators}"
)
@abstractmethod
def visit_operation(self, operation: Operation) -> Any:
"""Translate an Operation."""
@abstractmethod
def visit_comparison(self, comparison: Comparison) -> Any:
"""Translate a Comparison."""
@abstractmethod
def visit_structured_query(self, structured_query: StructuredQuery) -> Any:
"""Translate a StructuredQuery."""
def _to_snake_case(name: str) -> str:
"""Convert a name into snake_case."""
snake_case = ""
for i, char in enumerate(name):
if char.isupper() and i != 0:
snake_case += "_" + char.lower()
else:
snake_case += char.lower()
return snake_case
class Expr(BaseModel):
"""Base class for all expressions."""
def accept(self, visitor: Visitor) -> Any:
"""Accept a visitor.
Args:
visitor: visitor to accept
Returns:
result of visiting
"""
return getattr(visitor, f"visit_{_to_snake_case(self.__class__.__name__)}")(
self
)
class Operator(str, Enum):
"""Enumerator of the operations."""
AND = "and"
OR = "or"
NOT = "not"
class Comparator(str, Enum):
"""Enumerator of the comparison operators."""
EQ = "eq"
NE = "ne"
GT = "gt"
GTE = "gte"
LT = "lt"
LTE = "lte"
CONTAIN = "contain"
LIKE = "like"
IN = "in"
NIN = "nin"
class FilterDirective(Expr, ABC):
"""A filtering expression."""
class Comparison(FilterDirective):
"""A comparison to a value."""
comparator: Comparator
attribute: str
value: Any
def __init__(
self, comparator: Comparator, attribute: str, value: Any, **kwargs: Any
) -> None:
super().__init__(
comparator=comparator, attribute=attribute, value=value, **kwargs
)
class Operation(FilterDirective):
"""A logical operation over other directives."""
operator: Operator
arguments: List[FilterDirective]
def __init__(
self, operator: Operator, arguments: List[FilterDirective], **kwargs: Any
):
super().__init__(operator=operator, arguments=arguments, **kwargs)
class StructuredQuery(Expr):
"""A structured query."""
query: str
"""Query string."""
filter: Optional[FilterDirective]
"""Filtering expression."""
limit: Optional[int]
"""Limit on the number of results."""
def __init__(
self,
query: str,
filter: Optional[FilterDirective],
limit: Optional[int] = None,
**kwargs: Any,
):
super().__init__(query=query, filter=filter, limit=limit, **kwargs)
__all__ = [
"Visitor",
"Expr",
"Operator",
"Comparator",
"FilterDirective",
"Comparison",
"Operation",
"StructuredQuery",
]

View File

@ -17,7 +17,7 @@ except ImportError:
Transformer = object # type: ignore
Lark = object # type: ignore
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
FilterDirective,

View File

@ -1,7 +1,7 @@
"""Logic for converting internal query language to a valid AstraDB query."""
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -38,10 +38,10 @@ from langchain_core.language_models import BaseLanguageModel
from langchain_core.pydantic_v1 import Field, root_validator
from langchain_core.retrievers import BaseRetriever
from langchain_core.runnables import Runnable
from langchain_core.structured_query import StructuredQuery, Visitor
from langchain_core.vectorstores import VectorStore
from langchain.chains.query_constructor.base import load_query_constructor_runnable
from langchain.chains.query_constructor.ir import StructuredQuery, Visitor
from langchain.chains.query_constructor.schema import AttributeInfo
from langchain.retrievers.self_query.astradb import AstraDBTranslator
from langchain.retrievers.self_query.chroma import ChromaTranslator

View File

@ -1,6 +1,6 @@
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
"""Logic for converting internal query language to a valid DashVector query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -2,7 +2,7 @@ from collections import ChainMap
from itertools import chain
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
"""Logic for converting internal query language to a valid Chroma query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
"""Logic for converting internal query language to a valid Milvus query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
"""Logic for converting internal query language to a valid MongoDB Atlas query."""
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
import re
from typing import Any, Callable, Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -12,8 +12,7 @@ from langchain_community.vectorstores.redis.filters import (
RedisText,
)
from langchain_community.vectorstores.redis.schema import RedisModel
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Any, Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import Optional, Sequence, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,6 +1,6 @@
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -1,7 +1,7 @@
from datetime import datetime
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,

View File

@ -3,13 +3,13 @@ from typing import Any, cast
import lark
import pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
)
from langchain.chains.query_constructor.parser import get_parser
DEFAULT_PARSER = get_parser()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.astradb import AstraDBTranslator
DEFAULT_TRANSLATOR = AstraDBTranslator()

View File

@ -6,8 +6,7 @@ from langchain_core.callbacks.manager import (
CallbackManagerForRetrieverRun,
)
from langchain_core.documents import Document
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
@ -15,6 +14,7 @@ from langchain.chains.query_constructor.ir import (
StructuredQuery,
Visitor,
)
from langchain.chains.query_constructor.schema import AttributeInfo
from langchain.retrievers import SelfQueryRetriever
from tests.unit_tests.indexes.test_indexing import InMemoryVectorStore

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.chroma import ChromaTranslator
DEFAULT_TRANSLATOR = ChromaTranslator()

View File

@ -1,13 +1,13 @@
from typing import Any, Tuple
import pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
)
from langchain.retrievers.self_query.dashvector import DashvectorTranslator
DEFAULT_TRANSLATOR = DashvectorTranslator()

View File

@ -1,14 +1,14 @@
from typing import Any, Dict, Tuple
import pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.databricks_vector_search import (
DatabricksVectorSearchTranslator,
)

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.deeplake import DeepLakeTranslator
DEFAULT_TRANSLATOR = DeepLakeTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.dingo import DingoDBTranslator
DEFAULT_TRANSLATOR = DingoDBTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.elasticsearch import ElasticsearchTranslator
DEFAULT_TRANSLATOR = ElasticsearchTranslator()

View File

@ -1,14 +1,14 @@
from typing import Any, Dict, Tuple
import pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.milvus import MilvusTranslator
DEFAULT_TRANSLATOR = MilvusTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.mongodb_atlas import MongoDBAtlasTranslator
DEFAULT_TRANSLATOR = MongoDBAtlasTranslator()

View File

@ -1,14 +1,14 @@
from typing import Any, Dict, Tuple
import pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.myscale import MyScaleTranslator
DEFAULT_TRANSLATOR = MyScaleTranslator()

View File

@ -1,10 +1,11 @@
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.opensearch import OpenSearchTranslator
DEFAULT_TRANSLATOR = OpenSearchTranslator()

View File

@ -1,14 +1,14 @@
from typing import Dict, Tuple
import pytest as pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.pgvector import PGVectorTranslator
DEFAULT_TRANSLATOR = PGVectorTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.pinecone import PineconeTranslator
DEFAULT_TRANSLATOR = PineconeTranslator()

View File

@ -13,14 +13,14 @@ from langchain_community.vectorstores.redis.schema import (
TagFieldSchema,
TextFieldSchema,
)
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.redis import RedisTranslator

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.supabase import SupabaseVectorTranslator
DEFAULT_TRANSLATOR = SupabaseVectorTranslator()

View File

@ -1,10 +1,11 @@
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.tencentvectordb import TencentVectorDBTranslator

View File

@ -1,14 +1,14 @@
from typing import Dict, Tuple
import pytest as pytest
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.timescalevector import TimescaleVectorTranslator
DEFAULT_TRANSLATOR = TimescaleVectorTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.vectara import VectaraTranslator
DEFAULT_TRANSLATOR = VectaraTranslator()

View File

@ -1,12 +1,13 @@
from typing import Dict, Tuple
from langchain.chains.query_constructor.ir import (
from langchain_core.structured_query import (
Comparator,
Comparison,
Operation,
Operator,
StructuredQuery,
)
from langchain.retrievers.self_query.weaviate import WeaviateTranslator
DEFAULT_TRANSLATOR = WeaviateTranslator()