mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-01 11:02:37 +00:00
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
import datetime
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from langchain_community.document_loaders.base import BaseLoader
|
||||
|
||||
|
||||
def concatenate_rows(message: dict, title: str) -> str:
|
||||
"""
|
||||
Combine message information in a readable format ready to be used.
|
||||
Args:
|
||||
message: Message to be concatenated
|
||||
title: Title of the conversation
|
||||
|
||||
Returns:
|
||||
Concatenated message
|
||||
"""
|
||||
if not message:
|
||||
return ""
|
||||
|
||||
sender = message["author"]["role"] if message["author"] else "unknown"
|
||||
text = message["content"]["parts"][0]
|
||||
date = datetime.datetime.fromtimestamp(message["create_time"]).strftime(
|
||||
"%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
return f"{title} - {sender} on {date}: {text}\n\n"
|
||||
|
||||
|
||||
class ChatGPTLoader(BaseLoader):
|
||||
"""Load conversations from exported `ChatGPT` data."""
|
||||
|
||||
def __init__(self, log_file: str, num_logs: int = -1):
|
||||
"""Initialize a class object.
|
||||
|
||||
Args:
|
||||
log_file: Path to the log file
|
||||
num_logs: Number of logs to load. If 0, load all logs.
|
||||
"""
|
||||
self.log_file = log_file
|
||||
self.num_logs = num_logs
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
with open(self.log_file, encoding="utf8") as f:
|
||||
data = json.load(f)[: self.num_logs] if self.num_logs else json.load(f)
|
||||
|
||||
documents = []
|
||||
for d in data:
|
||||
title = d["title"]
|
||||
messages = d["mapping"]
|
||||
text = "".join(
|
||||
[
|
||||
concatenate_rows(messages[key]["message"], title)
|
||||
for idx, key in enumerate(messages)
|
||||
if not (
|
||||
idx == 0
|
||||
and messages[key]["message"]["author"]["role"] == "system"
|
||||
)
|
||||
]
|
||||
)
|
||||
metadata = {"source": str(self.log_file)}
|
||||
documents.append(Document(page_content=text, metadata=metadata))
|
||||
|
||||
return documents
|
Reference in New Issue
Block a user