github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-11 22:04:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

community[patch]: add VoyageEmbeddings truncation (#17638)

Browse Source
This commit is contained in:
Bagatur 2024-02-18 12:21:21 -05:00 committed by GitHub
parent d7c26c89b2
commit a058c8812d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
libs/community/langchain_community/embeddings/voyageai.py
View File

@ -86,6 +86,15 @@ class VoyageEmbeddings(BaseModel, Embeddings):
show_progress_bar: bool = False show_progress_bar: bool = False
"""Whether to show a progress bar when embedding. Must have tqdm installed if set """Whether to show a progress bar when embedding. Must have tqdm installed if set
to True.""" to True."""
truncation: Optional[bool] = None
"""Whether to truncate the input texts to fit within the context length.
If True, over-length input texts will be truncated to fit within the context
length, before vectorized by the embedding model. If False, an error will be
raised if any given text exceeds the context length. If not specified
(defaults to None), we will truncate the input text before sending it to the
embedding model if it slightly exceeds the context window length. If it
significantly exceeds the context window length, an error will be raised."""
class Config: class Config:
"""Configuration for this pydantic object.""" """Configuration for this pydantic object."""
@ -104,12 +113,14 @@ class VoyageEmbeddings(BaseModel, Embeddings):
self, input: List[str], input_type: Optional[str] = None self, input: List[str], input_type: Optional[str] = None
) -> Dict: ) -> Dict:
api_key = cast(SecretStr, self.voyage_api_key).get_secret_value() api_key = cast(SecretStr, self.voyage_api_key).get_secret_value()
params = { params: Dict = {
"url": self.voyage_api_base, "url": self.voyage_api_base,
"headers": {"Authorization": f"Bearer {api_key}"}, "headers": {"Authorization": f"Bearer {api_key}"},
"json": {"model": self.model, "input": input, "input_type": input_type}, "json": {"model": self.model, "input": input, "input_type": input_type},
"timeout": self.request_timeout, "timeout": self.request_timeout,
} }
if self.truncation is not None:
params["json"]["truncation"] = self.truncation
return params return params
def _get_embeddings( def _get_embeddings(