github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-16 22:51:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Support resource for dbgpts (#1527)

Browse Source
This commit is contained in:
Fangyin Cheng
2024-05-16 17:19:13 +08:00
committed by GitHub
parent 474d3166c7
commit 7138ec5166
7 changed files with 555 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbgpt/agent/expand/actions/tool_action.py
View File

@@ -125,7 +125,7 @@ class ToolAction(Action[ToolInput]):
"args": param.args,
"status": status,
"logo": None,
"result": tool_result,
"result": str(tool_result),
"err_msg": err_msg,
}
if not self.render_protocol:

18
dbgpt/agent/resource/manage.py
View File

@@ -99,6 +99,7 @@ class ResourceManager(BaseComponent):
resource_instance: Optional[Union[Resource, ToolResourceType]] = None,
resource_type: Optional[ResourceType] = None,
resource_type_alias: Optional[str] = None,
ignore_duplicate: bool = False,
):
"""Register a resource."""
if resource_instance and _is_function_tool(resource_instance):
@@ -117,14 +118,21 @@ class ResourceManager(BaseComponent):
resource_type=resource_type,
resource_type_alias=resource_type_alias,
)
if resource.key in self._resources:
if ignore_duplicate:
return
else:
raise ValueError(f"Resource {resource.key} already exists.")
self._resources[resource.key] = resource
self._type_to_resources[resource.type_unique_key].append(resource)
def get_supported_resources(
self, version: Optional[str] = None
) -> Dict[str, List[ParameterDescription]]:
) -> Dict[str, Union[List[ParameterDescription], List[str]]]:
"""Return the resources."""
results = {}
results: Dict[str, Union[List[ParameterDescription], List[str]]] = defaultdict(
list
)
for key, resource in self._resources.items():
parameter_class = resource.get_parameter_class()
resource_type = resource.type_unique_key
@@ -138,12 +146,14 @@ class ResourceManager(BaseComponent):
and isinstance(configs[0], ParameterDescription)
):
# v1, not compatible with class
configs = []
set_configs = set(results[resource_type])
if not resource.is_class:
for r in self._type_to_resources[resource_type]:
if not r.is_class:
configs.append(r.resource_instance.name) # type: ignore
set_configs.add(r.resource_instance.name) # type: ignore
configs = list(set_configs)
results[resource_type] = configs
return results
def build_resource_by_type(

5
dbgpt/util/dbgpts/base.py
View File

@@ -18,8 +18,8 @@ DEFAULT_REPO_MAP = {
"fangyinc/dbgpts": "https://github.com/fangyinc/dbgpts.git",
}
DEFAULT_PACKAGES = ["agents", "apps", "operators", "workflow"]
DEFAULT_PACKAGE_TYPES = ["agent", "app", "operator", "flow"]
DEFAULT_PACKAGES = ["agents", "apps", "operators", "workflow", "resources"]
DEFAULT_PACKAGE_TYPES = ["agent", "app", "operator", "flow", "resource"]
INSTALL_METADATA_FILE = "install_metadata.toml"
DBGPTS_METADATA_FILE = "dbgpts.toml"
@@ -28,6 +28,7 @@ TYPE_TO_PACKAGE = {
"app": "apps",
"operator": "operators",
"flow": "workflow",
"resource": "resources",
}

91
dbgpt/util/dbgpts/loader.py
View File

@@ -3,7 +3,7 @@ import logging
import os
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, TypeVar, cast
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, cast
import schedule
import tomlkit
@@ -73,8 +73,11 @@ class BasePackage(BaseModel):
@classmethod
def load_module_class(
cls, values: Dict[str, Any], expected_cls: Type[T]
) -> List[Type[T]]:
cls,
values: Dict[str, Any],
expected_cls: Type[T],
predicates: Optional[List[Callable[..., bool]]] = None,
) -> Tuple[List[Type[T]], List[Any]]:
import importlib.resources as pkg_resources
from dbgpt.core.awel.dag.loader import _load_modules_from_file
@@ -90,12 +93,15 @@ class BasePackage(BaseModel):
with pkg_resources.path(name, "__init__.py") as path:
mods = _load_modules_from_file(str(path), name, show_log=False)
all_cls = [_get_classes_from_module(m) for m in mods]
all_predicate_results = []
for m in mods:
all_predicate_results.extend(_get_from_module(m, predicates))
module_cls = []
for list_cls in all_cls:
for c in list_cls:
if issubclass(c, expected_cls):
module_cls.append(c)
return module_cls
return module_cls, all_predicate_results
class FlowPackage(BasePackage):
@@ -138,7 +144,7 @@ class OperatorPackage(BasePackage):
def build_from(cls, values: Dict[str, Any], ext_dict: Dict[str, Any]):
from dbgpt.core.awel import BaseOperator
values["operators"] = cls.load_module_class(values, BaseOperator)
values["operators"], _ = cls.load_module_class(values, BaseOperator)
return cls(**values)
@@ -153,7 +159,47 @@ class AgentPackage(BasePackage):
def build_from(cls, values: Dict[str, Any], ext_dict: Dict[str, Any]):
from dbgpt.agent import ConversableAgent
values["agents"] = cls.load_module_class(values, ConversableAgent)
values["agents"], _ = cls.load_module_class(values, ConversableAgent)
return cls(**values)
class ResourcePackage(BasePackage):
package_type: str = "resource"
resources: List[type] = Field(
default_factory=list, description="The resources of the package"
)
resource_instances: List[Any] = Field(
default_factory=list, description="The resource instances of the package"
)
@classmethod
def build_from(cls, values: Dict[str, Any], ext_dict: Dict[str, Any]):
from dbgpt.agent.resource import Resource
from dbgpt.agent.resource.tool.pack import _is_function_tool
def _predicate(obj):
if not obj:
return False
elif _is_function_tool(obj):
return True
elif isinstance(obj, Resource):
return True
elif isinstance(obj, type) and issubclass(obj, Resource):
return True
else:
return False
_, predicted_cls = cls.load_module_class(values, Resource, [_predicate])
resource_instances = []
resources = []
for o in predicted_cls:
if _is_function_tool(o) or isinstance(o, Resource):
resource_instances.append(o)
elif isinstance(o, type) and issubclass(o, Resource):
resources.append(o)
values["resource_instances"] = resource_instances
values["resources"] = resources
return cls(**values)
@@ -173,6 +219,17 @@ def _get_classes_from_module(module):
return classes
def _get_from_module(module, predicates: Optional[List[str]] = None):
if not predicates:
return []
results = []
for predicate in predicates:
for name, obj in inspect.getmembers(module, predicate):
if obj.__module__ == module.__name__:
results.append(obj)
return results
def _parse_package_metadata(package: InstalledPackage) -> BasePackage:
with open(
Path(package.root) / DBGPTS_METADATA_FILE, mode="r+", encoding="utf-8"
@@ -190,6 +247,9 @@ def _parse_package_metadata(package: InstalledPackage) -> BasePackage:
elif key == "agent":
pkg_dict = {k: v for k, v in value.items()}
pkg_dict["package_type"] = "agent"
elif key == "resource":
pkg_dict = {k: v for k, v in value.items()}
pkg_dict["package_type"] = "resource"
else:
ext_metadata[key] = value
pkg_dict["root"] = package.root
@@ -201,6 +261,8 @@ def _parse_package_metadata(package: InstalledPackage) -> BasePackage:
return OperatorPackage.build_from(pkg_dict, ext_metadata)
elif pkg_dict["package_type"] == "agent":
return AgentPackage.build_from(pkg_dict, ext_metadata)
elif pkg_dict["package_type"] == "resource":
return ResourcePackage.build_from(pkg_dict, ext_metadata)
else:
raise ValueError(
f"Unsupported package package_type: {pkg_dict['package_type']}"
@@ -316,3 +378,20 @@ class DBGPTsLoader(BaseComponent):
agent_manager.register_agent(agent_cls, ignore_duplicate=True)
except ValueError as e:
logger.warning(f"Register agent {agent_cls} error: {e}")
elif package.package_type == "resource":
from dbgpt.agent.resource import Resource
from dbgpt.agent.resource.manage import get_resource_manager
pkg = cast(ResourcePackage, package)
rm = get_resource_manager(self._system_app)
for inst in pkg.resource_instances:
try:
rm.register_resource(resource_instance=inst, ignore_duplicate=True)
except ValueError as e:
logger.warning(f"Register resource {inst} error: {e}")
for res in pkg.resources:
try:
if issubclass(res, Resource):
rm.register_resource(res, ignore_duplicate=True)
except ValueError as e:
logger.warning(f"Register resource {res} error: {e}")

56
dbgpt/util/dbgpts/template.py
View File

@@ -59,6 +59,15 @@ def create_template(
definition_type,
working_directory,
)
elif dbgpts_type == "resource":
_create_resource_template(
name,
mod_name,
dbgpts_type,
base_metadata,
definition_type,
working_directory,
)
else:
raise ValueError(f"Invalid dbgpts type: {dbgpts_type}")
@@ -145,6 +154,31 @@ def _create_agent_template(
_write_manifest_file(working_directory, name, mod_name)
def _create_resource_template(
name: str,
mod_name: str,
dbgpts_type: str,
base_metadata: dict,
definition_type: str,
working_directory: str,
):
json_dict = {
"resource": base_metadata,
"python_config": {},
"json_config": {},
}
if definition_type != "python":
raise click.ClickException(
f"Unsupported definition type: {definition_type} for dbgpts type: "
f"{dbgpts_type}"
)
_create_poetry_project(working_directory, name)
_write_dbgpts_toml(working_directory, name, json_dict)
_write_resource_init_file(working_directory, name, mod_name)
_write_manifest_file(working_directory, name, mod_name)
def _create_poetry_project(working_directory: str, name: str):
"""Create a new poetry project"""
@@ -365,3 +399,25 @@ if __name__ == "__main__":
"""
with open(init_file, "w") as f:
f.write(f'"""{name} agent package."""\n{content}')
def _write_resource_init_file(working_directory: str, name: str, mod_name: str):
"""Write the resource __init__.py file"""
init_file = Path(working_directory) / name / mod_name / "__init__.py"
content = """\"\"\"A custom resource module that provides a simple tool to send GET requests.\"\"\"
from dbgpt.agent.resource import tool
@tool
def simple_send_requests_get(url: str):
\"\"\"Send a GET request to the specified URL and return the text content.\"\"\"
import requests
response = requests.get(url)
return response.text
"""
with open(init_file, "w") as f:
f.write(content)