mirror of
https://github.com/hwchase17/langchain.git
synced 2025-04-27 11:41:51 +00:00
infra: update mypy 1.10, ruff 0.5 (#23721)
```python
"""python scripts/update_mypy_ruff.py"""
import glob
import tomllib
from pathlib import Path
import toml
import subprocess
import re
ROOT_DIR = Path(__file__).parents[1]
def main():
for path in glob.glob(str(ROOT_DIR / "libs/**/pyproject.toml"), recursive=True):
print(path)
with open(path, "rb") as f:
pyproject = tomllib.load(f)
try:
pyproject["tool"]["poetry"]["group"]["typing"]["dependencies"]["mypy"] = (
"^1.10"
)
pyproject["tool"]["poetry"]["group"]["lint"]["dependencies"]["ruff"] = (
"^0.5"
)
except KeyError:
continue
with open(path, "w") as f:
toml.dump(pyproject, f)
cwd = "/".join(path.split("/")[:-1])
completed = subprocess.run(
"poetry lock --no-update; poetry install --with typing; poetry run mypy . --no-color",
cwd=cwd,
shell=True,
capture_output=True,
text=True,
)
logs = completed.stdout.split("\n")
to_ignore = {}
for l in logs:
if re.match("^(.*)\:(\d+)\: error:.*\[(.*)\]", l):
path, line_no, error_type = re.match(
"^(.*)\:(\d+)\: error:.*\[(.*)\]", l
).groups()
if (path, line_no) in to_ignore:
to_ignore[(path, line_no)].append(error_type)
else:
to_ignore[(path, line_no)] = [error_type]
print(len(to_ignore))
for (error_path, line_no), error_types in to_ignore.items():
all_errors = ", ".join(error_types)
full_path = f"{cwd}/{error_path}"
try:
with open(full_path, "r") as f:
file_lines = f.readlines()
except FileNotFoundError:
continue
file_lines[int(line_no) - 1] = (
file_lines[int(line_no) - 1][:-1] + f" # type: ignore[{all_errors}]\n"
)
with open(full_path, "w") as f:
f.write("".join(file_lines))
subprocess.run(
"poetry run ruff format .; poetry run ruff --select I --fix .",
cwd=cwd,
shell=True,
capture_output=True,
text=True,
)
if __name__ == "__main__":
main()
```
This commit is contained in:
Bagatur
GitHub
parent
6cd56821dc
commit
a0c2281540
36
.github/actions/people/app/main.py
vendored
36
.github/actions/people/app/main.py
vendored
@ -350,11 +350,7 @@ def get_graphql_pr_edges(*, settings: Settings, after: Union[str, None] = None):
|
||||
print("Querying PRs...")
|
||||
else:
|
||||
print(f"Querying PRs with cursor {after}...")
|
||||
data = get_graphql_response(
|
||||
settings=settings,
|
||||
query=prs_query,
|
||||
after=after
|
||||
)
|
||||
data = get_graphql_response(settings=settings, query=prs_query, after=after)
|
||||
graphql_response = PRsResponse.model_validate(data)
|
||||
return graphql_response.data.repository.pullRequests.edges
|
||||
|
||||
@ -484,10 +480,16 @@ def get_contributors(settings: Settings):
|
||||
lines_changed = pr.additions + pr.deletions
|
||||
score = _logistic(files_changed, 20) + _logistic(lines_changed, 100)
|
||||
contributor_scores[pr.author.login] += score
|
||||
three_months_ago = (datetime.now(timezone.utc) - timedelta(days=3*30))
|
||||
three_months_ago = datetime.now(timezone.utc) - timedelta(days=3 * 30)
|
||||
if pr.createdAt > three_months_ago:
|
||||
recent_contributor_scores[pr.author.login] += score
|
||||
return contributors, contributor_scores, recent_contributor_scores, reviewers, authors
|
||||
return (
|
||||
contributors,
|
||||
contributor_scores,
|
||||
recent_contributor_scores,
|
||||
reviewers,
|
||||
authors,
|
||||
)
|
||||
|
||||
|
||||
def get_top_users(
|
||||
@ -524,9 +526,13 @@ if __name__ == "__main__":
|
||||
# question_commentors, question_last_month_commentors, question_authors = get_experts(
|
||||
# settings=settings
|
||||
# )
|
||||
contributors, contributor_scores, recent_contributor_scores, reviewers, pr_authors = get_contributors(
|
||||
settings=settings
|
||||
)
|
||||
(
|
||||
contributors,
|
||||
contributor_scores,
|
||||
recent_contributor_scores,
|
||||
reviewers,
|
||||
pr_authors,
|
||||
) = get_contributors(settings=settings)
|
||||
# authors = {**question_authors, **pr_authors}
|
||||
authors = {**pr_authors}
|
||||
maintainers_logins = {
|
||||
@ -559,7 +565,7 @@ if __name__ == "__main__":
|
||||
maintainers.append(
|
||||
{
|
||||
"login": login,
|
||||
"count": contributors[login], #+ question_commentors[login],
|
||||
"count": contributors[login], # + question_commentors[login],
|
||||
"avatarUrl": user.avatarUrl,
|
||||
"twitterUsername": user.twitterUsername,
|
||||
"url": user.url,
|
||||
@ -615,9 +621,7 @@ if __name__ == "__main__":
|
||||
new_people_content = yaml.dump(
|
||||
people, sort_keys=False, width=200, allow_unicode=True
|
||||
)
|
||||
if (
|
||||
people_old_content == new_people_content
|
||||
):
|
||||
if people_old_content == new_people_content:
|
||||
logging.info("The LangChain People data hasn't changed, finishing.")
|
||||
sys.exit(0)
|
||||
people_path.write_text(new_people_content, encoding="utf-8")
|
||||
@ -630,9 +634,7 @@ if __name__ == "__main__":
|
||||
logging.info(f"Creating a new branch {branch_name}")
|
||||
subprocess.run(["git", "checkout", "-B", branch_name], check=True)
|
||||
logging.info("Adding updated file")
|
||||
subprocess.run(
|
||||
["git", "add", str(people_path)], check=True
|
||||
)
|
||||
subprocess.run(["git", "add", str(people_path)], check=True)
|
||||
logging.info("Committing updated file")
|
||||
message = "👥 Update LangChain people data"
|
||||
result = subprocess.run(["git", "commit", "-m", message], check=True)
|
||||
|
||||
27
.github/scripts/check_diff.py
vendored
27
.github/scripts/check_diff.py
vendored
@ -1,11 +1,12 @@
|
||||
import glob
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, List, Set
|
||||
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
from collections import defaultdict
|
||||
import glob
|
||||
from typing import Dict, List, Set
|
||||
|
||||
|
||||
LANGCHAIN_DIRS = [
|
||||
"libs/core",
|
||||
@ -15,8 +16,13 @@ LANGCHAIN_DIRS = [
|
||||
"libs/experimental",
|
||||
]
|
||||
|
||||
|
||||
def all_package_dirs() -> Set[str]:
|
||||
return {"/".join(path.split("/")[:-1]) for path in glob.glob("./libs/**/pyproject.toml", recursive=True)}
|
||||
return {
|
||||
"/".join(path.split("/")[:-1]).lstrip("./")
|
||||
for path in glob.glob("./libs/**/pyproject.toml", recursive=True)
|
||||
if "libs/cli" not in path and "libs/standard-tests" not in path
|
||||
}
|
||||
|
||||
|
||||
def dependents_graph() -> dict:
|
||||
@ -26,9 +32,9 @@ def dependents_graph() -> dict:
|
||||
if "template" in path:
|
||||
continue
|
||||
with open(path, "rb") as f:
|
||||
pyproject = tomllib.load(f)['tool']['poetry']
|
||||
pyproject = tomllib.load(f)["tool"]["poetry"]
|
||||
pkg_dir = "libs" + "/".join(path.split("libs")[1].split("/")[:-1])
|
||||
for dep in pyproject['dependencies']:
|
||||
for dep in pyproject["dependencies"]:
|
||||
if "langchain" in dep:
|
||||
dependents[dep].add(pkg_dir)
|
||||
return dependents
|
||||
@ -122,9 +128,12 @@ if __name__ == "__main__":
|
||||
|
||||
outputs = {
|
||||
"dirs-to-lint": add_dependents(
|
||||
dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
|
||||
dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
|
||||
dependents,
|
||||
),
|
||||
"dirs-to-test": add_dependents(
|
||||
dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
|
||||
),
|
||||
"dirs-to-test": add_dependents(dirs_to_run["test"] | dirs_to_run["extended-test"], dependents),
|
||||
"dirs-to-extended-test": list(dirs_to_run["extended-test"]),
|
||||
"docs-edited": "true" if docs_edited else "",
|
||||
}
|
||||
|
||||
4
.github/scripts/get_min_versions.py
vendored
4
.github/scripts/get_min_versions.py
vendored
@ -74,6 +74,4 @@ if __name__ == "__main__":
|
||||
# Call the function to get the minimum versions
|
||||
min_versions = get_min_version_from_toml(toml_file)
|
||||
|
||||
print(
|
||||
" ".join([f"{lib}=={version}" for lib, version in min_versions.items()])
|
||||
)
|
||||
print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))
|
||||
|
||||
@ -48,14 +48,14 @@ lint lint_diff lint_package lint_tests:
|
||||
./scripts/check_pydantic.sh .
|
||||
./scripts/lint_imports.sh
|
||||
./scripts/check_pickle.sh .
|
||||
poetry run ruff .
|
||||
poetry run ruff check .
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I $(PYTHON_FILES)
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check --select I $(PYTHON_FILES)
|
||||
[ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) && poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
|
||||
|
||||
format format_diff:
|
||||
poetry run ruff format $(PYTHON_FILES)
|
||||
poetry run ruff --select I --fix $(PYTHON_FILES)
|
||||
poetry run ruff check --select I --fix $(PYTHON_FILES)
|
||||
|
||||
spell_check:
|
||||
poetry run codespell --toml pyproject.toml
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Main entrypoint into package."""
|
||||
|
||||
from importlib import metadata
|
||||
|
||||
try:
|
||||
|
||||
@ -206,8 +206,7 @@ class ChatCompletion:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[False] = False,
|
||||
**kwargs: Any,
|
||||
) -> dict:
|
||||
...
|
||||
) -> dict: ...
|
||||
|
||||
@overload
|
||||
@staticmethod
|
||||
@ -217,8 +216,7 @@ class ChatCompletion:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[True],
|
||||
**kwargs: Any,
|
||||
) -> Iterable:
|
||||
...
|
||||
) -> Iterable: ...
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
@ -249,8 +247,7 @@ class ChatCompletion:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[False] = False,
|
||||
**kwargs: Any,
|
||||
) -> dict:
|
||||
...
|
||||
) -> dict: ...
|
||||
|
||||
@overload
|
||||
@staticmethod
|
||||
@ -260,8 +257,7 @@ class ChatCompletion:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[True],
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator:
|
||||
...
|
||||
) -> AsyncIterator: ...
|
||||
|
||||
@staticmethod
|
||||
async def acreate(
|
||||
@ -319,8 +315,7 @@ class Completions:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[False] = False,
|
||||
**kwargs: Any,
|
||||
) -> ChatCompletions:
|
||||
...
|
||||
) -> ChatCompletions: ...
|
||||
|
||||
@overload
|
||||
@staticmethod
|
||||
@ -330,8 +325,7 @@ class Completions:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[True],
|
||||
**kwargs: Any,
|
||||
) -> Iterable:
|
||||
...
|
||||
) -> Iterable: ...
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
@ -366,8 +360,7 @@ class Completions:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[False] = False,
|
||||
**kwargs: Any,
|
||||
) -> ChatCompletions:
|
||||
...
|
||||
) -> ChatCompletions: ...
|
||||
|
||||
@overload
|
||||
@staticmethod
|
||||
@ -377,8 +370,7 @@ class Completions:
|
||||
provider: str = "ChatOpenAI",
|
||||
stream: Literal[True],
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator:
|
||||
...
|
||||
) -> AsyncIterator: ...
|
||||
|
||||
@staticmethod
|
||||
async def acreate(
|
||||
|
||||
@ -2189,14 +2189,14 @@ class AzureCosmosDBSemanticCache(BaseCache):
|
||||
index_name=index_name,
|
||||
)
|
||||
else:
|
||||
self._cache_dict[
|
||||
index_name
|
||||
] = AzureCosmosDBVectorSearch.from_connection_string(
|
||||
connection_string=self.cosmosdb_connection_string,
|
||||
namespace=namespace,
|
||||
embedding=self.embedding,
|
||||
index_name=index_name,
|
||||
application_name=self.application_name,
|
||||
self._cache_dict[index_name] = (
|
||||
AzureCosmosDBVectorSearch.from_connection_string(
|
||||
connection_string=self.cosmosdb_connection_string,
|
||||
namespace=namespace,
|
||||
embedding=self.embedding,
|
||||
index_name=index_name,
|
||||
application_name=self.application_name,
|
||||
)
|
||||
)
|
||||
|
||||
# create index for the vectorstore
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
BaseCallbackHandler --> <name>CallbackHandler # Example: AimCallbackHandler
|
||||
"""
|
||||
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
|
||||
@ -82,9 +82,9 @@ class ArizeCallbackHandler(BaseCallbackHandler):
|
||||
"completion_tokens", 0
|
||||
)
|
||||
else:
|
||||
self.prompt_tokens = (
|
||||
self.total_tokens
|
||||
) = self.completion_tokens = 0 # assign default value
|
||||
self.prompt_tokens = self.total_tokens = self.completion_tokens = (
|
||||
0 # assign default value
|
||||
)
|
||||
|
||||
for generations in response.generations:
|
||||
for generation in generations:
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""ArthurAI's Callback Handler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Callback handler for promptlayer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""A Tracer Implementation that records activity to Weights & Biases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@ -234,9 +235,9 @@ def build_tree(runs: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
|
||||
for child_id, parent_id in child_to_parent.items():
|
||||
parent_dict = id_to_data[parent_id]
|
||||
parent_dict[next(iter(parent_dict))][
|
||||
next(iter(id_to_data[child_id]))
|
||||
] = id_to_data[child_id][next(iter(id_to_data[child_id]))]
|
||||
parent_dict[next(iter(parent_dict))][next(iter(id_to_data[child_id]))] = (
|
||||
id_to_data[child_id][next(iter(id_to_data[child_id]))]
|
||||
)
|
||||
|
||||
root_dict = next(
|
||||
data for id_val, data in id_to_data.items() if id_val not in child_to_parent
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Methods for creating chains that use Ernie function-calling APIs."""
|
||||
|
||||
import inspect
|
||||
from typing import (
|
||||
Any,
|
||||
@ -191,9 +192,9 @@ def get_ernie_output_parser(
|
||||
}
|
||||
else:
|
||||
pydantic_schema = functions[0]
|
||||
output_parser: Union[
|
||||
BaseOutputParser, BaseGenerationOutputParser
|
||||
] = PydanticOutputFunctionsParser(pydantic_schema=pydantic_schema)
|
||||
output_parser: Union[BaseOutputParser, BaseGenerationOutputParser] = (
|
||||
PydanticOutputFunctionsParser(pydantic_schema=pydantic_schema)
|
||||
)
|
||||
else:
|
||||
output_parser = JsonOutputFunctionsParser(args_only=len(functions) <= 1)
|
||||
return output_parser
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
"""
|
||||
Question answering over an RDF or OWL graph using SPARQL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Question answering over a graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
"""
|
||||
Question answering over an RDF or OWL graph using SPARQL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Chain that hits a URL and then uses an LLM to parse results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Chain that makes API calls and summarizes the responses to answer a question."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Utilities for chat loaders."""
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Iterable, Iterator, List
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Astra DB - based chat message history, based on astrapy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Cassandra-based chat message history, based on cassIO."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Azure CosmosDB Memory History."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Firestore Chat Message History."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
""" Kafka-based chat message history by using confluent-kafka-python.
|
||||
confluent-kafka-python is under Apache 2.0 license.
|
||||
https://github.com/confluentinc/confluent-kafka-python
|
||||
"""Kafka-based chat message history by using confluent-kafka-python.
|
||||
confluent-kafka-python is under Apache 2.0 license.
|
||||
https://github.com/confluentinc/confluent-kafka-python
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Anyscale Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Azure OpenAI chat wrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""EverlyAI Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Fake ChatModel for testing purposes."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Google's PaLM Chat API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""JinaChat wrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""KonkoAI chat wrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around LiteLLM's model I/O library."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""LiteLLM Router as LangChain Model."""
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Moonshot chat models."""
|
||||
|
||||
from typing import Dict
|
||||
|
||||
from langchain_core.pydantic_v1 import root_validator
|
||||
|
||||
@ -27,28 +27,22 @@ CUSTOM_ENDPOINT_PREFIX = "ocid1.generativeaiendpoint"
|
||||
class Provider(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def stop_sequence_key(self) -> str:
|
||||
...
|
||||
def stop_sequence_key(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def chat_response_to_text(self, response: Any) -> str:
|
||||
...
|
||||
def chat_response_to_text(self, response: Any) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def chat_stream_to_text(self, event_data: Dict) -> str:
|
||||
...
|
||||
def chat_stream_to_text(self, event_data: Dict) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def chat_generation_info(self, response: Any) -> Dict[str, Any]:
|
||||
...
|
||||
def chat_generation_info(self, response: Any) -> Dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
def get_role(self, message: BaseMessage) -> str:
|
||||
...
|
||||
def get_role(self, message: BaseMessage) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def messages_to_oci_params(self, messages: Any) -> Dict[str, Any]:
|
||||
...
|
||||
def messages_to_oci_params(self, messages: Any) -> Dict[str, Any]: ...
|
||||
|
||||
|
||||
class CohereProvider(Provider):
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""OctoAI Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
|
||||
|
||||
from typing import Dict
|
||||
|
||||
from langchain_core.pydantic_v1 import Field, SecretStr, root_validator
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""PromptLayer wrapper."""
|
||||
|
||||
import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Google VertexAI chat-based models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around YandexGPT chat models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""ChatYuan2 wrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
|
||||
BaseCrossEncoder --> <name>CrossEncoder # Examples: SagemakerEndpointCrossEncoder
|
||||
"""
|
||||
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Interface to access to place that stores documents."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Simple in memory docstore in the form of a dict."""
|
||||
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
"""Wrapper around wikipedia API."""
|
||||
|
||||
|
||||
from typing import Union
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
@ -60,9 +60,9 @@ def fetch_mime_types(file_types: Sequence[_FileType]) -> Dict[str, str]:
|
||||
if file_type.value == "doc":
|
||||
mime_types_mapping[file_type.value] = "application/msword"
|
||||
elif file_type.value == "docx":
|
||||
mime_types_mapping[
|
||||
file_type.value
|
||||
] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" # noqa: E501
|
||||
mime_types_mapping[file_type.value] = (
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" # noqa: E501
|
||||
)
|
||||
elif file_type.value == "pdf":
|
||||
mime_types_mapping[file_type.value] = "application/pdf"
|
||||
return mime_types_mapping
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Use to load blobs from the local file system."""
|
||||
|
||||
import contextlib
|
||||
import mimetypes
|
||||
import tempfile
|
||||
|
||||
@ -5,7 +5,6 @@ from langchain_community.document_loaders.blob_loaders.schema import Blob, BlobL
|
||||
|
||||
|
||||
class YoutubeAudioLoader(BlobLoader):
|
||||
|
||||
"""Load YouTube urls as audio file(s)."""
|
||||
|
||||
def __init__(self, urls: List[str], save_dir: str):
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Load Documents from Docusarus Documentation"""
|
||||
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from langchain_community.document_loaders.sitemap import SitemapLoader
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
https://gist.github.com/foxmask/7b29c43a161e001ff04afdb2f181e31c
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from base64 import b64decode
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads Microsoft Excel files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loader that uses unstructured to load HTML files."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Iterator, List
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads .ipynb notebook files."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Union
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads data from OneDrive"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -4,6 +4,7 @@ You need to install two libraries to use this parser:
|
||||
pip install google-cloud-documentai
|
||||
pip install google-cloud-documentai-toolbox
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
This module contains some logic to help assemble more sophisticated parsers.
|
||||
"""
|
||||
|
||||
from typing import Iterator, Mapping, Optional
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Module contains common parsers for PDFs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Module includes a registry of default parser configurations."""
|
||||
|
||||
from langchain_community.document_loaders.base import BaseBlobParser
|
||||
from langchain_community.document_loaders.parsers.generic import MimeTypeBasedParser
|
||||
from langchain_community.document_loaders.parsers.msword import MsWordParser
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Module for parsing text files.."""
|
||||
|
||||
from typing import Iterator
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
@ -213,9 +213,9 @@ class PebbloSafeLoader(BaseLoader):
|
||||
if loading_end is True:
|
||||
payload["loading_end"] = "true"
|
||||
if "loader_details" in payload:
|
||||
payload["loader_details"][
|
||||
"source_aggregate_size"
|
||||
] = self.source_aggregate_size
|
||||
payload["loader_details"]["source_aggregate_size"] = (
|
||||
self.source_aggregate_size
|
||||
)
|
||||
payload = Doc(**payload).dict(exclude_unset=True)
|
||||
# Raw payload to be sent to classifier
|
||||
if self.classifier_location == "local":
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads RST files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads rich text files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Scrapfly Web Reader."""
|
||||
|
||||
import logging
|
||||
from typing import Iterator, List, Literal, Optional
|
||||
|
||||
|
||||
@ -61,9 +61,9 @@ class TensorflowDatasetLoader(BaseLoader):
|
||||
self.split_name: str = split_name
|
||||
self.load_max_docs = load_max_docs
|
||||
"""The maximum number of documents to load."""
|
||||
self.sample_to_document_function: Optional[
|
||||
Callable[[Dict], Document]
|
||||
] = sample_to_document_function
|
||||
self.sample_to_document_function: Optional[Callable[[Dict], Document]] = (
|
||||
sample_to_document_function
|
||||
)
|
||||
"""Custom function that transform a dataset sample into a Document."""
|
||||
|
||||
self._tfds_client = TensorflowDatasets( # type: ignore[call-arg]
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loader that uses unstructured to load files."""
|
||||
|
||||
import collections
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loader that uses unstructured to load HTML files."""
|
||||
|
||||
import logging
|
||||
from typing import Any, List
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
"""Loader that uses Playwright to load a page, then uses unstructured to load the html.
|
||||
"""
|
||||
"""Loader that uses Playwright to load a page, then uses unstructured to parse html."""
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, AsyncIterator, Dict, Iterator, List, Optional
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
"""Loader that uses Selenium to load a page, then uses unstructured to load the html.
|
||||
"""
|
||||
"""Loader that uses Selenium to load a page, then uses unstructured to load the html."""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, List, Literal, Optional, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Simple reader that reads weather data from OpenWeatherMap API"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads word documents."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from abc import ABC
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads Microsoft Excel files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Union
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Loads YouTube transcript."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Transform documents"""
|
||||
|
||||
from typing import Any, Callable, List, Sequence
|
||||
|
||||
import numpy as np
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Reorder documents"""
|
||||
|
||||
from typing import Any, List, Sequence
|
||||
|
||||
from langchain_core.documents import BaseDocumentTransformer, Document
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Document transformers that use OpenAI Functions models"""
|
||||
|
||||
from typing import Any, Dict, Optional, Sequence, Type, Union
|
||||
|
||||
from langchain_core.documents import BaseDocumentTransformer, Document
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Anyscale embeddings wrapper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
""" This file is for LLMRails Embedding """
|
||||
"""This file is for LLMRails Embedding"""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
@ -132,13 +132,13 @@ class OCIGenAIEmbeddings(BaseModel, Embeddings):
|
||||
oci_config=client_kwargs["config"]
|
||||
)
|
||||
elif values["auth_type"] == OCIAuthType(3).name:
|
||||
client_kwargs[
|
||||
"signer"
|
||||
] = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
||||
client_kwargs["signer"] = (
|
||||
oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
||||
)
|
||||
elif values["auth_type"] == OCIAuthType(4).name:
|
||||
client_kwargs[
|
||||
"signer"
|
||||
] = oci.auth.signers.get_resource_principals_signer()
|
||||
client_kwargs["signer"] = (
|
||||
oci.auth.signers.get_resource_principals_signer()
|
||||
)
|
||||
else:
|
||||
raise ValueError("Please provide valid value to auth_type")
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""HuggingFace sentence_transformer embedding models."""
|
||||
|
||||
from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings
|
||||
|
||||
SentenceTransformerEmbeddings = HuggingFaceEmbeddings
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around YandexGPT embedding models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
@ -105,15 +106,15 @@ class YandexGPTEmbeddings(BaseModel, Embeddings):
|
||||
if not values.get("doc_model_uri"):
|
||||
if values["folder_id"] == "":
|
||||
raise ValueError("'doc_model_uri' or 'folder_id' must be provided.")
|
||||
values[
|
||||
"doc_model_uri"
|
||||
] = f"emb://{values['folder_id']}/{values['doc_model_name']}/{values['model_version']}"
|
||||
values["doc_model_uri"] = (
|
||||
f"emb://{values['folder_id']}/{values['doc_model_name']}/{values['model_version']}"
|
||||
)
|
||||
if not values.get("model_uri"):
|
||||
if values["folder_id"] == "":
|
||||
raise ValueError("'model_uri' or 'folder_id' must be provided.")
|
||||
values[
|
||||
"model_uri"
|
||||
] = f"emb://{values['folder_id']}/{values['model_name']}/{values['model_version']}"
|
||||
values["model_uri"] = (
|
||||
f"emb://{values['folder_id']}/{values['model_name']}/{values['model_version']}"
|
||||
)
|
||||
if values["disable_request_logging"]:
|
||||
values["_grpc_metadata"].append(
|
||||
(
|
||||
|
||||
@ -6,6 +6,7 @@ There could be multiple strategies for selecting examples. For example, one coul
|
||||
select examples based on the similarity of the input to the examples. Another
|
||||
strategy could be to select examples based on the diversity of the examples.
|
||||
"""
|
||||
|
||||
from langchain_community.example_selectors.ngram_overlap import (
|
||||
NGramOverlapExampleSelector,
|
||||
ngram_overlap_score,
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
https://www.nltk.org/_modules/nltk/translate/bleu_score.html
|
||||
https://aclanthology.org/P02-1040.pdf
|
||||
"""
|
||||
|
||||
from typing import Dict, List
|
||||
|
||||
import numpy as np
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Networkx wrapper for graph operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, List, NamedTuple, Optional, Tuple
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Anyscale Endpoint"""
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
|
||||
@ -44,8 +44,7 @@ class _DatabricksClientBase(BaseModel, ABC):
|
||||
@abstractmethod
|
||||
def post(
|
||||
self, request: Any, transform_output_fn: Optional[Callable[..., str]] = None
|
||||
) -> Any:
|
||||
...
|
||||
) -> Any: ...
|
||||
|
||||
@property
|
||||
def llm(self) -> bool:
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around EdenAI's Generation API."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Konko AI's Completion API."""
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@ -39,12 +39,12 @@ class LayerupSecurity(LLM):
|
||||
response_guardrails: Optional[List[str]] = []
|
||||
mask: bool = False
|
||||
metadata: Optional[Dict[str, Any]] = {}
|
||||
handle_prompt_guardrail_violation: Callable[
|
||||
[dict], str
|
||||
] = default_guardrail_violation_handler
|
||||
handle_response_guardrail_violation: Callable[
|
||||
[dict], str
|
||||
] = default_guardrail_violation_handler
|
||||
handle_prompt_guardrail_violation: Callable[[dict], str] = (
|
||||
default_guardrail_violation_handler
|
||||
)
|
||||
handle_response_guardrail_violation: Callable[[dict], str] = (
|
||||
default_guardrail_violation_handler
|
||||
)
|
||||
client: Any #: :meta private:
|
||||
|
||||
@root_validator(pre=True)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Base interface for loading large language model APIs."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Union
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Wrapper around Minimax APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
@ -18,12 +18,10 @@ CUSTOM_ENDPOINT_PREFIX = "ocid1.generativeaiendpoint"
|
||||
class Provider(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def stop_sequence_key(self) -> str:
|
||||
...
|
||||
def stop_sequence_key(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def completion_response_to_text(self, response: Any) -> str:
|
||||
...
|
||||
def completion_response_to_text(self, response: Any) -> str: ...
|
||||
|
||||
|
||||
class CohereProvider(Provider):
|
||||
@ -144,13 +142,13 @@ class OCIGenAIBase(BaseModel, ABC):
|
||||
oci_config=client_kwargs["config"]
|
||||
)
|
||||
elif values["auth_type"] == OCIAuthType(3).name:
|
||||
client_kwargs[
|
||||
"signer"
|
||||
] = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
||||
client_kwargs["signer"] = (
|
||||
oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
||||
)
|
||||
elif values["auth_type"] == OCIAuthType(4).name:
|
||||
client_kwargs[
|
||||
"signer"
|
||||
] = oci.auth.signers.get_resource_principals_signer()
|
||||
client_kwargs["signer"] = (
|
||||
oci.auth.signers.get_resource_principals_signer()
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Please provide valid value to auth_type, "
|
||||
|
||||
@ -93,9 +93,9 @@ class OpenLLM(LLM):
|
||||
"""Keyword arguments to be passed to openllm.LLM"""
|
||||
|
||||
_runner: Optional[openllm.LLMRunner] = PrivateAttr(default=None)
|
||||
_client: Union[
|
||||
openllm.client.HTTPClient, openllm.client.GrpcClient, None
|
||||
] = PrivateAttr(default=None)
|
||||
_client: Union[openllm.client.HTTPClient, openllm.client.GrpcClient, None] = (
|
||||
PrivateAttr(default=None)
|
||||
)
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
@ -108,8 +108,7 @@ class OpenLLM(LLM):
|
||||
model_id: Optional[str] = ...,
|
||||
embedded: Literal[True, False] = ...,
|
||||
**llm_kwargs: Any,
|
||||
) -> None:
|
||||
...
|
||||
) -> None: ...
|
||||
|
||||
@overload
|
||||
def __init__(
|
||||
@ -118,8 +117,7 @@ class OpenLLM(LLM):
|
||||
server_url: str = ...,
|
||||
server_type: Literal["grpc", "http"] = ...,
|
||||
**llm_kwargs: Any,
|
||||
) -> None:
|
||||
...
|
||||
) -> None: ...
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
Based on https://github.com/saharNooby/rwkv.cpp/blob/master/rwkv/chat_with_bot.py
|
||||
https://github.com/BlinkDL/ChatRWKV/blob/main/v2/chat.py
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Mapping, Optional, Set
|
||||
|
||||
from langchain_core.callbacks import CallbackManagerForLLMRun
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user