github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-04-27 03:31:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

infra: update mypy 1.10, ruff 0.5 (#23721)

Browse Source 
```python
"""python scripts/update_mypy_ruff.py"""
import glob
import tomllib
from pathlib import Path

import toml
import subprocess
import re

ROOT_DIR = Path(__file__).parents[1]


def main():
    for path in glob.glob(str(ROOT_DIR / "libs/**/pyproject.toml"), recursive=True):
        print(path)
        with open(path, "rb") as f:
            pyproject = tomllib.load(f)
        try:
            pyproject["tool"]["poetry"]["group"]["typing"]["dependencies"]["mypy"] = (
                "^1.10"
            )
            pyproject["tool"]["poetry"]["group"]["lint"]["dependencies"]["ruff"] = (
                "^0.5"
            )
        except KeyError:
            continue
        with open(path, "w") as f:
            toml.dump(pyproject, f)
        cwd = "/".join(path.split("/")[:-1])
        completed = subprocess.run(
            "poetry lock --no-update; poetry install --with typing; poetry run mypy . --no-color",
            cwd=cwd,
            shell=True,
            capture_output=True,
            text=True,
        )
        logs = completed.stdout.split("\n")

        to_ignore = {}
        for l in logs:
            if re.match("^(.*)\:(\d+)\: error:.*\[(.*)\]", l):
                path, line_no, error_type = re.match(
                    "^(.*)\:(\d+)\: error:.*\[(.*)\]", l
                ).groups()
                if (path, line_no) in to_ignore:
                    to_ignore[(path, line_no)].append(error_type)
                else:
                    to_ignore[(path, line_no)] = [error_type]
        print(len(to_ignore))
        for (error_path, line_no), error_types in to_ignore.items():
            all_errors = ", ".join(error_types)
            full_path = f"{cwd}/{error_path}"
            try:
                with open(full_path, "r") as f:
                    file_lines = f.readlines()
            except FileNotFoundError:
                continue
            file_lines[int(line_no) - 1] = (
                file_lines[int(line_no) - 1][:-1] + f"  # type: ignore[{all_errors}]\n"
            )
            with open(full_path, "w") as f:
                f.write("".join(file_lines))

        subprocess.run(
            "poetry run ruff format .; poetry run ruff --select I --fix .",
            cwd=cwd,
            shell=True,
            capture_output=True,
            text=True,
        )


if __name__ == "__main__":
    main()

```
This commit is contained in:
Bagatur 2024-07-03 13:33:27 -04:00 committed by GitHub
parent 6cd56821dc
commit a0c2281540
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
915 changed files with 4759 additions and 4047 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.github/actions/people/app/main.py vendored
View File

@ -350,11 +350,7 @@ def get_graphql_pr_edges(*, settings: Settings, after: Union[str, None] = None):
print("Querying PRs...")
else:
print(f"Querying PRs with cursor {after}...")
data = get_graphql_response(
settings=settings,
query=prs_query,
after=after
)
data = get_graphql_response(settings=settings, query=prs_query, after=after)
graphql_response = PRsResponse.model_validate(data)
return graphql_response.data.repository.pullRequests.edges
@ -484,10 +480,16 @@ def get_contributors(settings: Settings):
lines_changed = pr.additions + pr.deletions
score = _logistic(files_changed, 20) + _logistic(lines_changed, 100)
contributor_scores[pr.author.login] += score
three_months_ago = (datetime.now(timezone.utc) - timedelta(days=3*30))
three_months_ago = datetime.now(timezone.utc) - timedelta(days=3 * 30)
if pr.createdAt > three_months_ago:
recent_contributor_scores[pr.author.login] += score
return contributors, contributor_scores, recent_contributor_scores, reviewers, authors
return (
contributors,
contributor_scores,
recent_contributor_scores,
reviewers,
authors,
)
def get_top_users(
@ -524,9 +526,13 @@ if __name__ == "__main__":
# question_commentors, question_last_month_commentors, question_authors = get_experts(
# settings=settings
# )
contributors, contributor_scores, recent_contributor_scores, reviewers, pr_authors = get_contributors(
settings=settings
)
(
contributors,
contributor_scores,
recent_contributor_scores,
reviewers,
pr_authors,
) = get_contributors(settings=settings)
# authors = {**question_authors, **pr_authors}
authors = {**pr_authors}
maintainers_logins = {
@ -559,7 +565,7 @@ if __name__ == "__main__":
maintainers.append(
{
"login": login,
"count": contributors[login], #+ question_commentors[login],
"count": contributors[login], # + question_commentors[login],
"avatarUrl": user.avatarUrl,
"twitterUsername": user.twitterUsername,
"url": user.url,
@ -615,9 +621,7 @@ if __name__ == "__main__":
new_people_content = yaml.dump(
people, sort_keys=False, width=200, allow_unicode=True
)
if (
people_old_content == new_people_content
):
if people_old_content == new_people_content:
logging.info("The LangChain People data hasn't changed, finishing.")
sys.exit(0)
people_path.write_text(new_people_content, encoding="utf-8")
@ -630,9 +634,7 @@ if __name__ == "__main__":
logging.info(f"Creating a new branch {branch_name}")
subprocess.run(["git", "checkout", "-B", branch_name], check=True)
logging.info("Adding updated file")
subprocess.run(
["git", "add", str(people_path)], check=True
)
subprocess.run(["git", "add", str(people_path)], check=True)
logging.info("Committing updated file")
message = "👥 Update LangChain people data"
result = subprocess.run(["git", "commit", "-m", message], check=True)
@ -641,4 +643,4 @@ if __name__ == "__main__":
logging.info("Creating PR")
pr = repo.create_pull(title=message, body=message, base="master", head=branch_name)
logging.info(f"Created PR: {pr.number}")
logging.info("Finished")
logging.info("Finished")

27
.github/scripts/check_diff.py vendored
View File

@ -1,11 +1,12 @@
import glob
import json
import sys
import os
from typing import Dict, List, Set
import re
import sys
import tomllib
from collections import defaultdict
import glob
from typing import Dict, List, Set
LANGCHAIN_DIRS = [
"libs/core",
@ -15,8 +16,13 @@ LANGCHAIN_DIRS = [
"libs/experimental",
]
def all_package_dirs() -> Set[str]:
return {"/".join(path.split("/")[:-1]) for path in glob.glob("./libs/**/pyproject.toml", recursive=True)}
return {
"/".join(path.split("/")[:-1]).lstrip("./")
for path in glob.glob("./libs/**/pyproject.toml", recursive=True)
if "libs/cli" not in path and "libs/standard-tests" not in path
}
def dependents_graph() -> dict:
@ -26,9 +32,9 @@ def dependents_graph() -> dict:
if "template" in path:
continue
with open(path, "rb") as f:
pyproject = tomllib.load(f)['tool']['poetry']
pyproject = tomllib.load(f)["tool"]["poetry"]
pkg_dir = "libs" + "/".join(path.split("libs")[1].split("/")[:-1])
for dep in pyproject['dependencies']:
for dep in pyproject["dependencies"]:
if "langchain" in dep:
dependents[dep].add(pkg_dir)
return dependents
@ -122,9 +128,12 @@ if __name__ == "__main__":
outputs = {
"dirs-to-lint": add_dependents(
dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
dependents,
),
"dirs-to-test": add_dependents(
dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
),
"dirs-to-test": add_dependents(dirs_to_run["test"] | dirs_to_run["extended-test"], dependents),
"dirs-to-extended-test": list(dirs_to_run["extended-test"]),
"docs-edited": "true" if docs_edited else "",
}

4
.github/scripts/get_min_versions.py vendored
View File

@ -74,6 +74,4 @@ if __name__ == "__main__":
# Call the function to get the minimum versions
min_versions = get_min_version_from_toml(toml_file)
print(
" ".join([f"{lib}=={version}" for lib, version in min_versions.items()])
)
print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))

6
libs/community/Makefile
View File

@ -48,14 +48,14 @@ lint lint_diff lint_package lint_tests:
./scripts/check_pydantic.sh .
./scripts/lint_imports.sh
./scripts/check_pickle.sh .
poetry run ruff .
poetry run ruff check .
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I $(PYTHON_FILES)
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff check --select I $(PYTHON_FILES)
[ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) && poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
format format_diff:
poetry run ruff format $(PYTHON_FILES)
poetry run ruff --select I --fix $(PYTHON_FILES)
poetry run ruff check --select I --fix $(PYTHON_FILES)
spell_check:
poetry run codespell --toml pyproject.toml

1
libs/community/langchain_community/__init__.py
View File

@ -1,4 +1,5 @@
"""Main entrypoint into package."""
from importlib import metadata
try:

24
libs/community/langchain_community/adapters/openai.py
View File

@ -206,8 +206,7 @@ class ChatCompletion:
provider: str = "ChatOpenAI",
stream: Literal[False] = False,
**kwargs: Any,
) -> dict:
...
) -> dict: ...
@overload
@staticmethod
@ -217,8 +216,7 @@ class ChatCompletion:
provider: str = "ChatOpenAI",
stream: Literal[True],
**kwargs: Any,
) -> Iterable:
...
) -> Iterable: ...
@staticmethod
def create(
@ -249,8 +247,7 @@ class ChatCompletion:
provider: str = "ChatOpenAI",
stream: Literal[False] = False,
**kwargs: Any,
) -> dict:
...
) -> dict: ...
@overload
@staticmethod
@ -260,8 +257,7 @@ class ChatCompletion:
provider: str = "ChatOpenAI",
stream: Literal[True],
**kwargs: Any,
) -> AsyncIterator:
...
) -> AsyncIterator: ...
@staticmethod
async def acreate(
@ -319,8 +315,7 @@ class Completions:
provider: str = "ChatOpenAI",
stream: Literal[False] = False,
**kwargs: Any,
) -> ChatCompletions:
...
) -> ChatCompletions: ...
@overload
@staticmethod
@ -330,8 +325,7 @@ class Completions:
provider: str = "ChatOpenAI",
stream: Literal[True],
**kwargs: Any,
) -> Iterable:
...
) -> Iterable: ...
@staticmethod
def create(
@ -366,8 +360,7 @@ class Completions:
provider: str = "ChatOpenAI",
stream: Literal[False] = False,
**kwargs: Any,
) -> ChatCompletions:
...
) -> ChatCompletions: ...
@overload
@staticmethod
@ -377,8 +370,7 @@ class Completions:
provider: str = "ChatOpenAI",
stream: Literal[True],
**kwargs: Any,
) -> AsyncIterator:
...
) -> AsyncIterator: ...
@staticmethod
async def acreate(

16
libs/community/langchain_community/cache.py
View File

@ -2189,14 +2189,14 @@ class AzureCosmosDBSemanticCache(BaseCache):
index_name=index_name,
)
else:
self._cache_dict[
index_name
] = AzureCosmosDBVectorSearch.from_connection_string(
connection_string=self.cosmosdb_connection_string,
namespace=namespace,
embedding=self.embedding,
index_name=index_name,
application_name=self.application_name,
self._cache_dict[index_name] = (
AzureCosmosDBVectorSearch.from_connection_string(
connection_string=self.cosmosdb_connection_string,
namespace=namespace,
embedding=self.embedding,
index_name=index_name,
application_name=self.application_name,
)
)
# create index for the vectorstore

1
libs/community/langchain_community/callbacks/__init__.py
View File

@ -6,6 +6,7 @@
BaseCallbackHandler --> <name>CallbackHandler # Example: AimCallbackHandler
"""
import importlib
from typing import TYPE_CHECKING, Any

6
libs/community/langchain_community/callbacks/arize_callback.py
View File

@ -82,9 +82,9 @@ class ArizeCallbackHandler(BaseCallbackHandler):
"completion_tokens", 0
)
else:
self.prompt_tokens = (
self.total_tokens
) = self.completion_tokens = 0 # assign default value
self.prompt_tokens = self.total_tokens = self.completion_tokens = (
0 # assign default value
)
for generations in response.generations:
for generation in generations:

1
libs/community/langchain_community/callbacks/arthur_callback.py
View File

@ -1,4 +1,5 @@
"""ArthurAI's Callback Handler."""
from __future__ import annotations
import os

1
libs/community/langchain_community/callbacks/promptlayer_callback.py
View File

@ -1,4 +1,5 @@
"""Callback handler for promptlayer."""
from __future__ import annotations
import datetime

7
libs/community/langchain_community/callbacks/tracers/wandb.py
View File

@ -1,4 +1,5 @@
"""A Tracer Implementation that records activity to Weights & Biases."""
from __future__ import annotations
import json
@ -234,9 +235,9 @@ def build_tree(runs: List[Dict[str, Any]]) -> Dict[str, Any]:
for child_id, parent_id in child_to_parent.items():
parent_dict = id_to_data[parent_id]
parent_dict[next(iter(parent_dict))][
next(iter(id_to_data[child_id]))
] = id_to_data[child_id][next(iter(id_to_data[child_id]))]
parent_dict[next(iter(parent_dict))][next(iter(id_to_data[child_id]))] = (
id_to_data[child_id][next(iter(id_to_data[child_id]))]
)
root_dict = next(
data for id_val, data in id_to_data.items() if id_val not in child_to_parent

7
libs/community/langchain_community/chains/ernie_functions/base.py
View File

@ -1,4 +1,5 @@
"""Methods for creating chains that use Ernie function-calling APIs."""
import inspect
from typing import (
Any,
@ -191,9 +192,9 @@ def get_ernie_output_parser(
}
else:
pydantic_schema = functions[0]
output_parser: Union[
BaseOutputParser, BaseGenerationOutputParser
] = PydanticOutputFunctionsParser(pydantic_schema=pydantic_schema)
output_parser: Union[BaseOutputParser, BaseGenerationOutputParser] = (
PydanticOutputFunctionsParser(pydantic_schema=pydantic_schema)
)
else:
output_parser = JsonOutputFunctionsParser(args_only=len(functions) <= 1)
return output_parser

1
libs/community/langchain_community/chains/graph_qa/arangodb.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
import re

1
libs/community/langchain_community/chains/graph_qa/base.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/cypher.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
import re

1
libs/community/langchain_community/chains/graph_qa/falkordb.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
import re

1
libs/community/langchain_community/chains/graph_qa/gremlin.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/hugegraph.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/kuzu.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
import re

1
libs/community/langchain_community/chains/graph_qa/nebulagraph.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/neptune_sparql.py
View File

@ -1,6 +1,7 @@
"""
Question answering over an RDF or OWL graph using SPARQL.
"""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/ontotext_graphdb.py
View File

@ -1,4 +1,5 @@
"""Question answering over a graph."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Dict, List, Optional

1
libs/community/langchain_community/chains/graph_qa/sparql.py
View File

@ -1,6 +1,7 @@
"""
Question answering over an RDF or OWL graph using SPARQL.
"""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/llm_requests.py
View File

@ -1,4 +1,5 @@
"""Chain that hits a URL and then uses an LLM to parse results."""
from __future__ import annotations
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chains/openapi/chain.py
View File

@ -1,4 +1,5 @@
"""Chain that makes API calls and summarizes the responses to answer a question."""
from __future__ import annotations
import json

1
libs/community/langchain_community/chat_loaders/utils.py
View File

@ -1,4 +1,5 @@
"""Utilities for chat loaders."""
from copy import deepcopy
from typing import Iterable, Iterator, List

1
libs/community/langchain_community/chat_message_histories/astradb.py
View File

@ -1,4 +1,5 @@
"""Astra DB - based chat message history, based on astrapy."""
from __future__ import annotations
import json

1
libs/community/langchain_community/chat_message_histories/cassandra.py
View File

@ -1,4 +1,5 @@
"""Cassandra-based chat message history, based on cassIO."""
from __future__ import annotations
import json

1
libs/community/langchain_community/chat_message_histories/cosmos_db.py
View File

@ -1,4 +1,5 @@
"""Azure CosmosDB Memory History."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_message_histories/firestore.py
View File

@ -1,4 +1,5 @@
"""Firestore Chat Message History."""
from __future__ import annotations
import logging

7
libs/community/langchain_community/chat_message_histories/kafka.py
View File

@ -1,7 +1,8 @@
""" Kafka-based chat message history by using confluent-kafka-python.
confluent-kafka-python is under Apache 2.0 license.
https://github.com/confluentinc/confluent-kafka-python
"""Kafka-based chat message history by using confluent-kafka-python.
confluent-kafka-python is under Apache 2.0 license.
https://github.com/confluentinc/confluent-kafka-python
"""
from __future__ import annotations
import json

1
libs/community/langchain_community/chat_models/anyscale.py
View File

@ -1,4 +1,5 @@
"""Anyscale Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/azure_openai.py
View File

@ -1,4 +1,5 @@
"""Azure OpenAI chat wrapper."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/everlyai.py
View File

@ -1,4 +1,5 @@
"""EverlyAI Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/fake.py
View File

@ -1,4 +1,5 @@
"""Fake ChatModel for testing purposes."""
import asyncio
import time
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Union

1
libs/community/langchain_community/chat_models/google_palm.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Google's PaLM Chat API."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/jinachat.py
View File

@ -1,4 +1,5 @@
"""JinaChat wrapper."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/konko.py
View File

@ -1,4 +1,5 @@
"""KonkoAI chat wrapper."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/litellm.py
View File

@ -1,4 +1,5 @@
"""Wrapper around LiteLLM's model I/O library."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/litellm_router.py
View File

@ -1,4 +1,5 @@
"""LiteLLM Router as LangChain Model."""
from typing import (
Any,
AsyncIterator,

1
libs/community/langchain_community/chat_models/moonshot.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Moonshot chat models."""
from typing import Dict
from langchain_core.pydantic_v1 import root_validator

18
libs/community/langchain_community/chat_models/oci_generative_ai.py
View File

@ -27,28 +27,22 @@ CUSTOM_ENDPOINT_PREFIX = "ocid1.generativeaiendpoint"
class Provider(ABC):
@property
@abstractmethod
def stop_sequence_key(self) -> str:
...
def stop_sequence_key(self) -> str: ...
@abstractmethod
def chat_response_to_text(self, response: Any) -> str:
...
def chat_response_to_text(self, response: Any) -> str: ...
@abstractmethod
def chat_stream_to_text(self, event_data: Dict) -> str:
...
def chat_stream_to_text(self, event_data: Dict) -> str: ...
@abstractmethod
def chat_generation_info(self, response: Any) -> Dict[str, Any]:
...
def chat_generation_info(self, response: Any) -> Dict[str, Any]: ...
@abstractmethod
def get_role(self, message: BaseMessage) -> str:
...
def get_role(self, message: BaseMessage) -> str: ...
@abstractmethod
def messages_to_oci_params(self, messages: Any) -> Dict[str, Any]:
...
def messages_to_oci_params(self, messages: Any) -> Dict[str, Any]: ...
class CohereProvider(Provider):

1
libs/community/langchain_community/chat_models/octoai.py
View File

@ -1,4 +1,5 @@
"""OctoAI Endpoints chat wrapper. Relies heavily on ChatOpenAI."""
from typing import Dict
from langchain_core.pydantic_v1 import Field, SecretStr, root_validator

1
libs/community/langchain_community/chat_models/promptlayer_openai.py
View File

@ -1,4 +1,5 @@
"""PromptLayer wrapper."""
import datetime
from typing import Any, Dict, List, Optional

1
libs/community/langchain_community/chat_models/vertexai.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Google VertexAI chat-based models."""
from __future__ import annotations
import base64

1
libs/community/langchain_community/chat_models/yandex.py
View File

@ -1,4 +1,5 @@
"""Wrapper around YandexGPT chat models."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/chat_models/yuan2.py
View File

@ -1,4 +1,5 @@
"""ChatYuan2 wrapper."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/cross_encoders/__init__.py
View File

@ -9,6 +9,7 @@
BaseCrossEncoder --> <name>CrossEncoder # Examples: SagemakerEndpointCrossEncoder
"""
import importlib
from typing import TYPE_CHECKING, Any

1
libs/community/langchain_community/docstore/base.py
View File

@ -1,4 +1,5 @@
"""Interface to access to place that stores documents."""
from abc import ABC, abstractmethod
from typing import Dict, List, Union

1
libs/community/langchain_community/docstore/in_memory.py
View File

@ -1,4 +1,5 @@
"""Simple in memory docstore in the form of a dict."""
from typing import Dict, List, Optional, Union
from langchain_core.documents import Document

1
libs/community/langchain_community/docstore/wikipedia.py
View File

@ -1,6 +1,5 @@
"""Wrapper around wikipedia API."""
from typing import Union
from langchain_core.documents import Document

6
libs/community/langchain_community/document_loaders/base_o365.py
View File

@ -60,9 +60,9 @@ def fetch_mime_types(file_types: Sequence[_FileType]) -> Dict[str, str]:
if file_type.value == "doc":
mime_types_mapping[file_type.value] = "application/msword"
elif file_type.value == "docx":
mime_types_mapping[
file_type.value
] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" # noqa: E501
mime_types_mapping[file_type.value] = (
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" # noqa: E501
)
elif file_type.value == "pdf":
mime_types_mapping[file_type.value] = "application/pdf"
return mime_types_mapping

1
libs/community/langchain_community/document_loaders/blob_loaders/cloud_blob_loader.py
View File

@ -1,4 +1,5 @@
"""Use to load blobs from the local file system."""
import contextlib
import mimetypes
import tempfile

1
libs/community/langchain_community/document_loaders/blob_loaders/youtube_audio.py
View File

@ -5,7 +5,6 @@ from langchain_community.document_loaders.blob_loaders.schema import Blob, BlobL
class YoutubeAudioLoader(BlobLoader):
"""Load YouTube urls as audio file(s)."""
def __init__(self, urls: List[str], save_dir: str):

1
libs/community/langchain_community/document_loaders/docusaurus.py
View File

@ -1,4 +1,5 @@
"""Load Documents from Docusarus Documentation"""
from typing import Any, List, Optional
from langchain_community.document_loaders.sitemap import SitemapLoader

1
libs/community/langchain_community/document_loaders/evernote.py
View File

@ -2,6 +2,7 @@
https://gist.github.com/foxmask/7b29c43a161e001ff04afdb2f181e31c
"""
import hashlib
import logging
from base64 import b64decode

1
libs/community/langchain_community/document_loaders/excel.py
View File

@ -1,4 +1,5 @@
"""Loads Microsoft Excel files."""
from pathlib import Path
from typing import Any, List, Union

1
libs/community/langchain_community/document_loaders/news.py
View File

@ -1,4 +1,5 @@
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, Iterator, List

1
libs/community/langchain_community/document_loaders/notebook.py
View File

@ -1,4 +1,5 @@
"""Loads .ipynb notebook files."""
import json
from pathlib import Path
from typing import Any, List, Union

1
libs/community/langchain_community/document_loaders/onedrive.py
View File

@ -1,4 +1,5 @@
"""Loads data from OneDrive"""
from __future__ import annotations
import logging

1
libs/community/langchain_community/document_loaders/parsers/docai.py
View File

@ -4,6 +4,7 @@ You need to install two libraries to use this parser:
pip install google-cloud-documentai
pip install google-cloud-documentai-toolbox
"""
import logging
import re
import time

1
libs/community/langchain_community/document_loaders/parsers/generic.py
View File

@ -2,6 +2,7 @@
This module contains some logic to help assemble more sophisticated parsers.
"""
from typing import Iterator, Mapping, Optional
from langchain_core.documents import Document

1
libs/community/langchain_community/document_loaders/parsers/pdf.py
View File

@ -1,4 +1,5 @@
"""Module contains common parsers for PDFs."""
from __future__ import annotations
import warnings

1
libs/community/langchain_community/document_loaders/parsers/registry.py
View File

@ -1,4 +1,5 @@
"""Module includes a registry of default parser configurations."""
from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.parsers.generic import MimeTypeBasedParser
from langchain_community.document_loaders.parsers.msword import MsWordParser

1
libs/community/langchain_community/document_loaders/parsers/txt.py
View File

@ -1,4 +1,5 @@
"""Module for parsing text files.."""
from typing import Iterator
from langchain_core.documents import Document

6
libs/community/langchain_community/document_loaders/pebblo.py
View File

@ -213,9 +213,9 @@ class PebbloSafeLoader(BaseLoader):
if loading_end is True:
payload["loading_end"] = "true"
if "loader_details" in payload:
payload["loader_details"][
"source_aggregate_size"
] = self.source_aggregate_size
payload["loader_details"]["source_aggregate_size"] = (
self.source_aggregate_size
)
payload = Doc(**payload).dict(exclude_unset=True)
# Raw payload to be sent to classifier
if self.classifier_location == "local":

1
libs/community/langchain_community/document_loaders/rst.py
View File

@ -1,4 +1,5 @@
"""Loads RST files."""
from pathlib import Path
from typing import Any, List, Union

1
libs/community/langchain_community/document_loaders/rtf.py
View File

@ -1,4 +1,5 @@
"""Loads rich text files."""
from pathlib import Path
from typing import Any, List, Union

1
libs/community/langchain_community/document_loaders/scrapfly.py
View File

@ -1,4 +1,5 @@
"""Scrapfly Web Reader."""
import logging
from typing import Iterator, List, Literal, Optional

6
libs/community/langchain_community/document_loaders/tensorflow_datasets.py
View File

@ -61,9 +61,9 @@ class TensorflowDatasetLoader(BaseLoader):
self.split_name: str = split_name
self.load_max_docs = load_max_docs
"""The maximum number of documents to load."""
self.sample_to_document_function: Optional[
Callable[[Dict], Document]
] = sample_to_document_function
self.sample_to_document_function: Optional[Callable[[Dict], Document]] = (
sample_to_document_function
)
"""Custom function that transform a dataset sample into a Document."""
self._tfds_client = TensorflowDatasets( # type: ignore[call-arg]

1
libs/community/langchain_community/document_loaders/unstructured.py
View File

@ -1,4 +1,5 @@
"""Loader that uses unstructured to load files."""
import collections
from abc import ABC, abstractmethod
from pathlib import Path

1
libs/community/langchain_community/document_loaders/url.py
View File

@ -1,4 +1,5 @@
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, List

4
libs/community/langchain_community/document_loaders/url_playwright.py
View File

@ -1,5 +1,5 @@
"""Loader that uses Playwright to load a page, then uses unstructured to load the html.
"""
"""Loader that uses Playwright to load a page, then uses unstructured to parse html."""
import logging
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, AsyncIterator, Dict, Iterator, List, Optional

4
libs/community/langchain_community/document_loaders/url_selenium.py
View File

@ -1,5 +1,5 @@
"""Loader that uses Selenium to load a page, then uses unstructured to load the html.
"""
"""Loader that uses Selenium to load a page, then uses unstructured to load the html."""
import logging
from typing import TYPE_CHECKING, List, Literal, Optional, Union

1
libs/community/langchain_community/document_loaders/weather.py
View File

@ -1,4 +1,5 @@
"""Simple reader that reads weather data from OpenWeatherMap API"""
from __future__ import annotations
from datetime import datetime

1
libs/community/langchain_community/document_loaders/word_document.py
View File

@ -1,4 +1,5 @@
"""Loads word documents."""
import os
import tempfile
from abc import ABC

1
libs/community/langchain_community/document_loaders/xml.py
View File

@ -1,4 +1,5 @@
"""Loads Microsoft Excel files."""
from pathlib import Path
from typing import Any, List, Union

1
libs/community/langchain_community/document_loaders/youtube.py
View File

@ -1,4 +1,5 @@
"""Loads YouTube transcript."""
from __future__ import annotations
import logging

1
libs/community/langchain_community/document_transformers/embeddings_redundant_filter.py
View File

@ -1,4 +1,5 @@
"""Transform documents"""
from typing import Any, Callable, List, Sequence
import numpy as np

1
libs/community/langchain_community/document_transformers/long_context_reorder.py
View File

@ -1,4 +1,5 @@
"""Reorder documents"""
from typing import Any, List, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document

1
libs/community/langchain_community/document_transformers/openai_functions.py
View File

@ -1,4 +1,5 @@
"""Document transformers that use OpenAI Functions models"""
from typing import Any, Dict, Optional, Sequence, Type, Union
from langchain_core.documents import BaseDocumentTransformer, Document

1
libs/community/langchain_community/embeddings/anyscale.py
View File

@ -1,4 +1,5 @@
"""Anyscale embeddings wrapper."""
from __future__ import annotations
from typing import Dict

3
libs/community/langchain_community/embeddings/llm_rails.py
View File

@ -1,4 +1,5 @@
""" This file is for LLMRails Embedding """
"""This file is for LLMRails Embedding"""
from typing import Dict, List, Optional
import requests

12
libs/community/langchain_community/embeddings/oci_generative_ai.py
View File

@ -132,13 +132,13 @@ class OCIGenAIEmbeddings(BaseModel, Embeddings):
oci_config=client_kwargs["config"]
)
elif values["auth_type"] == OCIAuthType(3).name:
client_kwargs[
"signer"
] = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
client_kwargs["signer"] = (
oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
)
elif values["auth_type"] == OCIAuthType(4).name:
client_kwargs[
"signer"
] = oci.auth.signers.get_resource_principals_signer()
client_kwargs["signer"] = (
oci.auth.signers.get_resource_principals_signer()
)
else:
raise ValueError("Please provide valid value to auth_type")

1
libs/community/langchain_community/embeddings/sentence_transformer.py
View File

@ -1,4 +1,5 @@
"""HuggingFace sentence_transformer embedding models."""
from langchain_community.embeddings.huggingface import HuggingFaceEmbeddings
SentenceTransformerEmbeddings = HuggingFaceEmbeddings

13
libs/community/langchain_community/embeddings/yandex.py
View File

@ -1,4 +1,5 @@
"""Wrapper around YandexGPT embedding models."""
from __future__ import annotations
import logging
@ -105,15 +106,15 @@ class YandexGPTEmbeddings(BaseModel, Embeddings):
if not values.get("doc_model_uri"):
if values["folder_id"] == "":
raise ValueError("'doc_model_uri' or 'folder_id' must be provided.")
values[
"doc_model_uri"
] = f"emb://{values['folder_id']}/{values['doc_model_name']}/{values['model_version']}"
values["doc_model_uri"] = (
f"emb://{values['folder_id']}/{values['doc_model_name']}/{values['model_version']}"
)
if not values.get("model_uri"):
if values["folder_id"] == "":
raise ValueError("'model_uri' or 'folder_id' must be provided.")
values[
"model_uri"
] = f"emb://{values['folder_id']}/{values['model_name']}/{values['model_version']}"
values["model_uri"] = (
f"emb://{values['folder_id']}/{values['model_name']}/{values['model_version']}"
)
if values["disable_request_logging"]:
values["_grpc_metadata"].append(
(

1
libs/community/langchain_community/example_selectors/__init__.py
View File

@ -6,6 +6,7 @@ There could be multiple strategies for selecting examples. For example, one coul
select examples based on the similarity of the input to the examples. Another
strategy could be to select examples based on the diversity of the examples.
"""
from langchain_community.example_selectors.ngram_overlap import (
NGramOverlapExampleSelector,
ngram_overlap_score,

1
libs/community/langchain_community/example_selectors/ngram_overlap.py
View File

@ -3,6 +3,7 @@
https://www.nltk.org/_modules/nltk/translate/bleu_score.html
https://aclanthology.org/P02-1040.pdf
"""
from typing import Dict, List
import numpy as np

1
libs/community/langchain_community/graphs/networkx_graph.py
View File

@ -1,4 +1,5 @@
"""Networkx wrapper for graph operations."""
from __future__ import annotations
from typing import Any, List, NamedTuple, Optional, Tuple

1
libs/community/langchain_community/llms/anyscale.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Anyscale Endpoint"""
from typing import (
Any,
Dict,

3
libs/community/langchain_community/llms/databricks.py
View File

@ -44,8 +44,7 @@ class _DatabricksClientBase(BaseModel, ABC):
@abstractmethod
def post(
self, request: Any, transform_output_fn: Optional[Callable[..., str]] = None
) -> Any:
...
) -> Any: ...
@property
def llm(self) -> bool:

1
libs/community/langchain_community/llms/edenai.py
View File

@ -1,4 +1,5 @@
"""Wrapper around EdenAI's Generation API."""
import logging
from typing import Any, Dict, List, Literal, Optional

1
libs/community/langchain_community/llms/konko.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Konko AI's Completion API."""
import logging
import warnings
from typing import Any, Dict, List, Optional

12
libs/community/langchain_community/llms/layerup_security.py
View File

@ -39,12 +39,12 @@ class LayerupSecurity(LLM):
response_guardrails: Optional[List[str]] = []
mask: bool = False
metadata: Optional[Dict[str, Any]] = {}
handle_prompt_guardrail_violation: Callable[
[dict], str
] = default_guardrail_violation_handler
handle_response_guardrail_violation: Callable[
[dict], str
] = default_guardrail_violation_handler
handle_prompt_guardrail_violation: Callable[[dict], str] = (
default_guardrail_violation_handler
)
handle_response_guardrail_violation: Callable[[dict], str] = (
default_guardrail_violation_handler
)
client: Any #: :meta private:
@root_validator(pre=True)

1
libs/community/langchain_community/llms/loading.py
View File

@ -1,4 +1,5 @@
"""Base interface for loading large language model APIs."""
import json
from pathlib import Path
from typing import Any, Union

1
libs/community/langchain_community/llms/minimax.py
View File

@ -1,4 +1,5 @@
"""Wrapper around Minimax APIs."""
from __future__ import annotations
import logging

18
libs/community/langchain_community/llms/oci_generative_ai.py
View File

@ -18,12 +18,10 @@ CUSTOM_ENDPOINT_PREFIX = "ocid1.generativeaiendpoint"
class Provider(ABC):
@property
@abstractmethod
def stop_sequence_key(self) -> str:
...
def stop_sequence_key(self) -> str: ...
@abstractmethod
def completion_response_to_text(self, response: Any) -> str:
...
def completion_response_to_text(self, response: Any) -> str: ...
class CohereProvider(Provider):
@ -144,13 +142,13 @@ class OCIGenAIBase(BaseModel, ABC):
oci_config=client_kwargs["config"]
)
elif values["auth_type"] == OCIAuthType(3).name:
client_kwargs[
"signer"
] = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
client_kwargs["signer"] = (
oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
)
elif values["auth_type"] == OCIAuthType(4).name:
client_kwargs[
"signer"
] = oci.auth.signers.get_resource_principals_signer()
client_kwargs["signer"] = (
oci.auth.signers.get_resource_principals_signer()
)
else:
raise ValueError(
"Please provide valid value to auth_type, "

12
libs/community/langchain_community/llms/openllm.py
View File

@ -93,9 +93,9 @@ class OpenLLM(LLM):
"""Keyword arguments to be passed to openllm.LLM"""
_runner: Optional[openllm.LLMRunner] = PrivateAttr(default=None)
_client: Union[
openllm.client.HTTPClient, openllm.client.GrpcClient, None
] = PrivateAttr(default=None)
_client: Union[openllm.client.HTTPClient, openllm.client.GrpcClient, None] = (
PrivateAttr(default=None)
)
class Config:
extra = "forbid"
@ -108,8 +108,7 @@ class OpenLLM(LLM):
model_id: Optional[str] = ...,
embedded: Literal[True, False] = ...,
**llm_kwargs: Any,
) -> None:
...
) -> None: ...
@overload
def __init__(
@ -118,8 +117,7 @@ class OpenLLM(LLM):
server_url: str = ...,
server_type: Literal["grpc", "http"] = ...,
**llm_kwargs: Any,
) -> None:
...
) -> None: ...
def __init__(
self,

1
libs/community/langchain_community/llms/rwkv.py
View File

@ -3,6 +3,7 @@
Based on https://github.com/saharNooby/rwkv.cpp/blob/master/rwkv/chat_with_bot.py
https://github.com/BlinkDL/ChatRWKV/blob/main/v2/chat.py
"""
from typing import Any, Dict, List, Mapping, Optional, Set
from langchain_core.callbacks import CallbackManagerForLLMRun

Some files were not shown because too many files have changed in this diff Show More