mirror of
https://github.com/hwchase17/langchain.git
synced 2025-11-27 06:00:56 +00:00
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463)
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import ast
|
||||
from typing import Any, List
|
||||
|
||||
from langchain_community.document_loaders.parsers.language.code_segmenter import (
|
||||
CodeSegmenter,
|
||||
)
|
||||
|
||||
|
||||
class PythonSegmenter(CodeSegmenter):
|
||||
"""Code segmenter for `Python`."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.source_lines = self.code.splitlines()
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
try:
|
||||
ast.parse(self.code)
|
||||
return True
|
||||
except SyntaxError:
|
||||
return False
|
||||
|
||||
def _extract_code(self, node: Any) -> str:
|
||||
start = node.lineno - 1
|
||||
end = node.end_lineno
|
||||
return "\n".join(self.source_lines[start:end])
|
||||
|
||||
def extract_functions_classes(self) -> List[str]:
|
||||
tree = ast.parse(self.code)
|
||||
functions_classes = []
|
||||
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
functions_classes.append(self._extract_code(node))
|
||||
|
||||
return functions_classes
|
||||
|
||||
def simplify_code(self) -> str:
|
||||
tree = ast.parse(self.code)
|
||||
simplified_lines = self.source_lines[:]
|
||||
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
start = node.lineno - 1
|
||||
simplified_lines[start] = f"# Code for: {simplified_lines[start]}"
|
||||
|
||||
assert isinstance(node.end_lineno, int)
|
||||
for line_num in range(start + 1, node.end_lineno):
|
||||
simplified_lines[line_num] = None # type: ignore
|
||||
|
||||
return "\n".join(line for line in simplified_lines if line is not None)
|
||||
Reference in New Issue
Block a user