github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-09-19 17:36:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Other File Utilities (#3209)

Browse Source 
Add other File Utilities, include
- List Directory
- Search for file
- Move
- Copy
- Remove file

Bundle as toolkit
Add a notebook that connects to the Chat Agent, which somewhat supports
multi-arg input tools
Update original read/write files to return the original dir paths and
better handle unsupported file paths.
Add unit tests
This commit is contained in:
Zander Chase
2023-04-28 10:53:37 -07:00
committed by GitHub
parent 491c27f861
commit 334c162f16
21 changed files with 848 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
langchain/tools/__init__.py
View File

@@ -3,6 +3,13 @@
from langchain.tools.base import BaseTool
from langchain.tools.bing_search.tool import BingSearchResults, BingSearchRun
from langchain.tools.ddg_search.tool import DuckDuckGoSearchResults, DuckDuckGoSearchRun
from langchain.tools.file_management.copy import CopyFileTool
from langchain.tools.file_management.delete import DeleteFileTool
from langchain.tools.file_management.file_search import FileSearchTool
from langchain.tools.file_management.list_dir import ListDirectoryTool
from langchain.tools.file_management.move import MoveFileTool
from langchain.tools.file_management.read import ReadFileTool
from langchain.tools.file_management.write import WriteFileTool
from langchain.tools.google_places.tool import GooglePlacesTool
from langchain.tools.google_search.tool import GoogleSearchResults, GoogleSearchRun
from langchain.tools.ifttt import IFTTTWebhook
@@ -21,25 +28,33 @@ from langchain.tools.playwright import (
from langchain.tools.plugin import AIPluginTool
__all__ = [
"APIOperation",
"AIPluginTool",
"BaseBrowserTool",
"BaseTool",
"BaseTool",
"BingSearchResults",
"BingSearchRun",
"ClickTool",
"CopyFileTool",
"CurrentWebPageTool",
"DeleteFileTool",
"DuckDuckGoSearchResults",
"DuckDuckGoSearchRun",
"DuckDuckGoSearchRun",
"ExtractHyperlinksTool",
"ExtractTextTool",
"FileSearchTool",
"GetElementsTool",
"GooglePlacesTool",
"GoogleSearchResults",
"GoogleSearchRun",
"IFTTTWebhook",
"ListDirectoryTool",
"MoveFileTool",
"NavigateBackTool",
"NavigateTool",
"OpenAPISpec",
"AIPluginTool",
"ReadFileTool",
"WriteFileTool",
"APIOperation",
]

19
langchain/tools/file_management/__init__.py
View File

@@ -0,0 +1,19 @@
"""File Management Tools."""
from langchain.tools.file_management.copy import CopyFileTool
from langchain.tools.file_management.delete import DeleteFileTool
from langchain.tools.file_management.file_search import FileSearchTool
from langchain.tools.file_management.list_dir import ListDirectoryTool
from langchain.tools.file_management.move import MoveFileTool
from langchain.tools.file_management.read import ReadFileTool
from langchain.tools.file_management.write import WriteFileTool
__all__ = [
"CopyFileTool",
"DeleteFileTool",
"FileSearchTool",
"MoveFileTool",
"ReadFileTool",
"WriteFileTool",
"ListDirectoryTool",
]

46
langchain/tools/file_management/copy.py Normal file
View File

@@ -0,0 +1,46 @@
import shutil
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class FileCopyInput(BaseModel):
"""Input for CopyFileTool."""
source_path: str = Field(..., description="Path of the file to copy")
destination_path: str = Field(..., description="Path to save the copied file")
class CopyFileTool(BaseFileTool):
name: str = "copy_file"
args_schema: Type[BaseModel] = FileCopyInput
description: str = "Create a copy of a file in a specified location"
def _run(self, source_path: str, destination_path: str) -> str:
try:
source_path_ = self.get_relative_path(source_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(
arg_name="source_path", value=source_path
)
try:
destination_path_ = self.get_relative_path(destination_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(
arg_name="destination_path", value=destination_path
)
try:
shutil.copy2(source_path_, destination_path_, follow_symlinks=False)
return f"File copied successfully from {source_path} to {destination_path}."
except Exception as e:
return "Error: " + str(e)
async def _arun(self, source_path: str, destination_path: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

39
langchain/tools/file_management/delete.py Normal file
View File

@@ -0,0 +1,39 @@
import os
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class FileDeleteInput(BaseModel):
"""Input for DeleteFileTool."""
file_path: str = Field(..., description="Path of the file to delete")
class DeleteFileTool(BaseFileTool):
name: str = "file_delete"
args_schema: Type[BaseModel] = FileDeleteInput
description: str = "Delete a file"
def _run(self, file_path: str) -> str:
try:
file_path_ = self.get_relative_path(file_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(arg_name="file_path", value=file_path)
if not file_path_.exists():
return f"Error: no such file or directory: {file_path}"
try:
os.remove(file_path_)
return f"File deleted successfully: {file_path}."
except Exception as e:
return "Error: " + str(e)
async def _arun(self, file_path: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

55
langchain/tools/file_management/file_search.py Normal file
View File

@@ -0,0 +1,55 @@
import fnmatch
import os
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class FileSearchInput(BaseModel):
"""Input for FileSearchTool."""
dir_path: str = Field(
default=".",
description="Subdirectory to search in.",
)
pattern: str = Field(
...,
description="Unix shell regex, where * matches everything.",
)
class FileSearchTool(BaseFileTool):
name: str = "file_search"
args_schema: Type[BaseModel] = FileSearchInput
description: str = (
"Recursively search for files in a subdirectory that match the regex pattern"
)
def _run(self, pattern: str, dir_path: str = ".") -> str:
try:
dir_path_ = self.get_relative_path(dir_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(arg_name="dir_path", value=dir_path)
matches = []
try:
for root, _, filenames in os.walk(dir_path_):
for filename in fnmatch.filter(filenames, pattern):
absolute_path = os.path.join(root, filename)
relative_path = os.path.relpath(absolute_path, dir_path_)
matches.append(relative_path)
if matches:
return "\n".join(matches)
else:
return f"No files found for pattern {pattern} in directory {dir_path}"
except Exception as e:
return "Error: " + str(e)
async def _arun(self, dir_path: str, pattern: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

40
langchain/tools/file_management/list_dir.py Normal file
View File

@@ -0,0 +1,40 @@
import os
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class DirectoryListingInput(BaseModel):
"""Input for ListDirectoryTool."""
dir_path: str = Field(default=".", description="Subdirectory to list.")
class ListDirectoryTool(BaseFileTool):
name: str = "list_directory"
args_schema: Type[BaseModel] = DirectoryListingInput
description: str = "List files and directories in a specified folder"
def _run(self, dir_path: str = ".") -> str:
try:
dir_path_ = self.get_relative_path(dir_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(arg_name="dir_path", value=dir_path)
try:
entries = os.listdir(dir_path_)
if entries:
return "\n".join(entries)
else:
return f"No files found in directory {dir_path}"
except Exception as e:
return "Error: " + str(e)
async def _arun(self, dir_path: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

49
langchain/tools/file_management/move.py Normal file
View File

@@ -0,0 +1,49 @@
import shutil
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class FileMoveInput(BaseModel):
"""Input for MoveFileTool."""
source_path: str = Field(..., description="Path of the file to move")
destination_path: str = Field(..., description="New path for the moved file")
class MoveFileTool(BaseFileTool):
name: str = "move_file"
args_schema: Type[BaseModel] = FileMoveInput
description: str = "Move or rename a file from one location to another"
def _run(self, source_path: str, destination_path: str) -> str:
try:
source_path_ = self.get_relative_path(source_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(
arg_name="source_path", value=source_path
)
try:
destination_path_ = self.get_relative_path(destination_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(
arg_name="destination_path_", value=destination_path_
)
if not source_path_.exists():
return f"Error: no such file or directory {source_path}"
try:
# shutil.move expects str args in 3.8
shutil.move(str(source_path_), destination_path_)
return f"File moved successfully from {source_path} to {destination_path}."
except Exception as e:
return "Error: " + str(e)
async def _arun(self, source_path: str, destination_path: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

29
langchain/tools/file_management/read.py
View File

@@ -1,10 +1,12 @@
from pathlib import Path
from typing import Optional, Type
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.base import BaseTool
from langchain.tools.file_management.utils import get_validated_relative_path
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class ReadFileInput(BaseModel):
@@ -13,21 +15,18 @@ class ReadFileInput(BaseModel):
file_path: str = Field(..., description="name of file")
class ReadFileTool(BaseTool):
class ReadFileTool(BaseFileTool):
name: str = "read_file"
args_schema: Type[BaseModel] = ReadFileInput
description: str = "Read file from disk"
root_dir: Optional[str] = None
"""Directory to read file from.
If specified, raises an error for file_paths oustide root_dir."""
def _run(self, file_path: str) -> str:
read_path = (
get_validated_relative_path(Path(self.root_dir), file_path)
if self.root_dir
else Path(file_path)
)
try:
read_path = self.get_relative_path(file_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(arg_name="file_path", value=file_path)
if not read_path.exists():
return f"Error: no such file or directory: {file_path}"
try:
with read_path.open("r", encoding="utf-8") as f:
content = f.read()
@@ -35,6 +34,6 @@ class ReadFileTool(BaseTool):
except Exception as e:
return "Error: " + str(e)
async def _arun(self, tool_input: str) -> str:
async def _arun(self, file_path: str) -> str:
# TODO: Add aiofiles method
raise NotImplementedError

38
langchain/tools/file_management/utils.py
View File

@@ -1,5 +1,10 @@
import sys
from pathlib import Path
from typing import Any, Optional
from pydantic import BaseModel
from langchain.tools.base import BaseTool
def is_relative_to(path: Path, root: Path) -> bool:
@@ -14,6 +19,35 @@ def is_relative_to(path: Path, root: Path) -> bool:
return False
INVALID_PATH_TEMPLATE = (
"Error: Access denied to {arg_name}: {value}."
" Permission granted exclusively to the current working directory"
)
class FileValidationError(ValueError):
"""Error for paths outside the root directory."""
class BaseFileTool(BaseTool, BaseModel):
"""Input for ReadFileTool."""
root_dir: Optional[str] = None
"""The final path will be chosen relative to root_dir if specified."""
def get_relative_path(self, file_path: str) -> Path:
"""Get the relative path, returning an error if unsupported."""
if self.root_dir is None:
return Path(file_path)
return get_validated_relative_path(Path(self.root_dir), file_path)
def _run(self, *args: Any, **kwargs: Any) -> str:
raise NotImplementedError
async def _arun(self, *args: Any, **kwargs: Any) -> str:
raise NotImplementedError
def get_validated_relative_path(root: Path, user_path: str) -> Path:
"""Resolve a relative path, raising an error if not within the root directory."""
# Note, this still permits symlinks from outside that point within the root.
@@ -22,5 +56,7 @@ def get_validated_relative_path(root: Path, user_path: str) -> Path:
full_path = (root / user_path).resolve()
if not is_relative_to(full_path, root):
raise ValueError(f"Path {user_path} is outside of the allowed directory {root}")
raise FileValidationError(
f"Path {user_path} is outside of the allowed directory {root}"
)
return full_path

35
langchain/tools/file_management/write.py
View File

@@ -1,10 +1,12 @@
from pathlib import Path
from typing import Optional, Type
from typing import Type
from pydantic import BaseModel, Field
from langchain.tools.base import BaseTool
from langchain.tools.file_management.utils import get_validated_relative_path
from langchain.tools.file_management.utils import (
INVALID_PATH_TEMPLATE,
BaseFileTool,
FileValidationError,
)
class WriteFileInput(BaseModel):
@@ -12,31 +14,30 @@ class WriteFileInput(BaseModel):
file_path: str = Field(..., description="name of file")
text: str = Field(..., description="text to write to file")
append: bool = Field(
default=False, description="Whether to append to an existing file."
)
class WriteFileTool(BaseTool):
class WriteFileTool(BaseFileTool):
name: str = "write_file"
args_schema: Type[BaseModel] = WriteFileInput
description: str = "Write file to disk"
root_dir: Optional[str] = None
"""Directory to write file to.
If specified, raises an error for file_paths oustide root_dir."""
def _run(self, file_path: str, text: str) -> str:
write_path = (
get_validated_relative_path(Path(self.root_dir), file_path)
if self.root_dir
else Path(file_path)
)
def _run(self, file_path: str, text: str, append: bool = False) -> str:
try:
write_path = self.get_relative_path(file_path)
except FileValidationError:
return INVALID_PATH_TEMPLATE.format(arg_name="file_path", value=file_path)
try:
write_path.parent.mkdir(exist_ok=True, parents=False)
with write_path.open("w", encoding="utf-8") as f:
mode = "a" if append else "w"
with write_path.open(mode, encoding="utf-8") as f:
f.write(text)
return f"File written successfully to {file_path}."
except Exception as e:
return "Error: " + str(e)
async def _arun(self, file_path: str, text: str) -> str:
async def _arun(self, file_path: str, text: str, append: bool = False) -> str:
# TODO: Add aiofiles method
raise NotImplementedError