mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-01 11:02:37 +00:00
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
This commit is contained in:
47
libs/community/langchain_community/document_loaders/toml.py
Normal file
47
libs/community/langchain_community/document_loaders/toml.py
Normal file
@@ -0,0 +1,47 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Iterator, List, Union
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from langchain_community.document_loaders.base import BaseLoader
|
||||
|
||||
|
||||
class TomlLoader(BaseLoader):
|
||||
"""Load `TOML` files.
|
||||
|
||||
It can load a single source file or several files in a single
|
||||
directory.
|
||||
"""
|
||||
|
||||
def __init__(self, source: Union[str, Path]):
|
||||
"""Initialize the TomlLoader with a source file or directory."""
|
||||
self.source = Path(source)
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Load and return all documents."""
|
||||
return list(self.lazy_load())
|
||||
|
||||
def lazy_load(self) -> Iterator[Document]:
|
||||
"""Lazily load the TOML documents from the source file or directory."""
|
||||
import tomli
|
||||
|
||||
if self.source.is_file() and self.source.suffix == ".toml":
|
||||
files = [self.source]
|
||||
elif self.source.is_dir():
|
||||
files = list(self.source.glob("**/*.toml"))
|
||||
else:
|
||||
raise ValueError("Invalid source path or file type")
|
||||
|
||||
for file_path in files:
|
||||
with file_path.open("r", encoding="utf-8") as file:
|
||||
content = file.read()
|
||||
try:
|
||||
data = tomli.loads(content)
|
||||
doc = Document(
|
||||
page_content=json.dumps(data),
|
||||
metadata={"source": str(file_path)},
|
||||
)
|
||||
yield doc
|
||||
except tomli.TOMLDecodeError as e:
|
||||
print(f"Error parsing TOML file {file_path}: {e}")
|
Reference in New Issue
Block a user