github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-11 22:09:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dbgpts): dbgpts module add

Browse Source
This commit is contained in:
yhjun1026
2024-08-23 14:48:05 +08:00
parent 1133ec170d
commit 2ccedaee6e
11 changed files with 201 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dbgpt/app/initialization/serve_initialization.py
View File

@@ -75,3 +75,14 @@ def register_serve_apps(system_app: SystemApp, cfg: Config):
# Register serve feedback
system_app.register(FeedbackServe)
# ################################ Chat Feedback Register End ########################################
# ################################ DbGpts Register Begin ########################################
# Register serve dbgptshub
from dbgpt.serve.dbgpts.hub.serve import Serve as DbgptsHubServe
system_app.register(DbgptsHubServe)
# Register serve dbgptsmy
from dbgpt.serve.dbgpts.my.serve import Serve as DbgptsMyServe
system_app.register(DbgptsMyServe)
# ################################ DbGpts Register End ########################################

49
dbgpt/serve/dbgpts/hub/api/endpoints.py
View File

@@ -1,3 +1,4 @@
import logging
from functools import cache
from typing import List, Optional
@@ -18,6 +19,8 @@ router = APIRouter()
global_system_app: Optional[SystemApp] = None
logger = logging.getLogger(__name__)
def get_service() -> Service:
"""Get the service instance"""
@@ -168,7 +171,51 @@ async def query_page(
Returns:
ServerResponse: The response
"""
return Result.succ(service.get_list_by_page(request, page, page_size))
try:
return Result.succ(service.get_list_by_page(request, page, page_size))
except Exception as e:
logger.exception("query_page exception!")
return Result.failed(msg=str(e))
@router.post("/source/refresh", response_model=Result[str])
async def source_refresh(
service: Service = Depends(get_service),
):
logger.info(f"source_refresh")
try:
await service.refresh_hub_from_git()
return Result.succ(None)
except Exception as e:
logger.error("Dbgpts hub source refresh Error!", e)
return Result.failed(err_code="E0020", msg=f"Dbgpts Hub refresh Error! {e}")
@router.post("/install", response_model=Result[str])
async def install(request: ServeRequest):
logger.info(f"dbgpts install:{request.name},{request.type}")
try:
from dbgpt.serve.dbgpts.my.config import (
SERVE_SERVICE_COMPONENT_NAME as MY_GPTS_SERVICE_COMPONENT,
)
from dbgpt.serve.dbgpts.my.service.service import Service as MyGptsService
mygpts_service: MyGptsService = global_system_app.get_component(
MY_GPTS_SERVICE_COMPONENT, MyGptsService
)
await mygpts_service.install_gpts(
name=request.name,
type=request.type,
repo=request.storage_channel,
dbgpt_path=request.storage_url,
user_code=None,
sys_code=None,
)
return Result.succ(None)
except Exception as e:
logger.error("Plugin Install Error!", e)
return Result.failed(err_code="E0021", msg=f"Plugin Install Error {e}")
def init_endpoints(system_app: SystemApp) -> None:

13
dbgpt/serve/dbgpts/hub/api/schemas.py
View File

@@ -10,17 +10,16 @@ class ServeRequest(BaseModel):
"""DbgptsHub request model"""
id: Optional[int] = Field(None, description="id")
name: Optional[str] = Field(..., description="Dbgpts name")
description: Optional[str] = Field(..., description="Dbgpts description")
author: Optional[str] = Field(None, description="Dbgpts author")
email: Optional[str] = Field(None, description="Dbgpts email")
name: Optional[str] = Field(None, description="Dbgpts name")
type: Optional[str] = Field(None, description="Dbgpts type")
version: Optional[str] = Field(None, description="Dbgpts version")
description: Optional[str] = Field(None, description="Dbgpts description")
author: Optional[str] = Field(None, description="Dbgpts author")
email: Optional[str] = Field(None, description="Dbgpts email")
storage_channel: Optional[str] = Field(None, description="Dbgpts storage channel")
storage_url: Optional[str] = Field(None, description="Dbgpts storage url")
download_param: Optional[str] = Field(None, description="Dbgpts download param")
installed: Optional[int] = Field(None, description="Dbgpts installed")
gmt_created: Optional[str] = Field(None, description="Dbgpts upload time")
model_config = ConfigDict(title=f"ServeRequest for {SERVE_APP_NAME_HUMP}")
@@ -29,4 +28,6 @@ class ServeRequest(BaseModel):
return model_to_dict(self, **kwargs)
ServerResponse = ServeRequest
class ServerResponse(ServeRequest):
gmt_created: Optional[str] = Field(None, description="Dbgpts create time")
gmt_modified: Optional[str] = Field(None, description="Dbgpts upload time")

17
dbgpt/serve/dbgpts/hub/models/models.py
View File

@@ -30,7 +30,12 @@ class ServeEntity(Model):
default=datetime.utcnow,
comment="plugin upload time",
)
gmt_modified = Column(DateTime, default=datetime.now, comment="Record update time")
gmt_modified = Column(
DateTime,
default=datetime.now,
onupdate=datetime.utcnow,
comment="Record update time",
)
installed = Column(Integer, default=False, comment="plugin already installed count")
UniqueConstraint("name", "type", name="uk_dbgpts")
@@ -84,7 +89,6 @@ class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
storage_url=entity.storage_url,
download_param=entity.download_param,
installed=entity.installed,
gmt_created=entity.gmt_created,
)
def to_response(self, entity: ServeEntity) -> ServerResponse:
@@ -96,5 +100,12 @@ class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
Returns:
RES: The response
"""
gmt_created_str = entity.gmt_created.strftime("%Y-%m-%d %H:%M:%S")
gmt_modified_str = entity.gmt_modified.strftime("%Y-%m-%d %H:%M:%S")
request = self.to_request(entity)
return self.to_request(entity)
return ServerResponse(
**request.to_dict(),
gmt_created=gmt_created_str,
gmt_modified=gmt_modified_str,
)

30
dbgpt/serve/dbgpts/hub/service/service.py
View File

@@ -120,7 +120,17 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
Returns:
List[ServerResponse]: The response
"""
query_request = request
query_request = ServeRequest(
name=request.name,
type=request.type,
version=request.version,
description=request.description,
author=request.author,
storage_channel=request.storage_channel,
storage_url=request.storage_url,
installed=request.installed,
)
return self.dao.get_list_page(query_request, page, page_size)
async def refresh_hub_from_git(
@@ -142,12 +152,22 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
try:
for repo, package, name, gpts_path in data:
old_hub_info = self.dao.get_one(ServeRequest(name=name, type=package))
if not name:
logger.info(
f"dbgpts error repo:{repo}, package:{package}, name:{name}, gpts_path:{gpts_path}"
)
continue
old_hub_info = self.get(ServeRequest(name=name, type=package))
if (
get_repo_path(repo)
== "/Users/tuyang.yhj/.dbgpts/repos/eosphoros/dbgpts/dbgpts.toml"
):
logger.info("xxx")
base_package: BasePackage = parse_package_metadata(
InstalledPackage(
name=name,
repo=repo,
root=get_repo_path(repo),
root=str(gpts_path),
package=package,
)
)
@@ -166,7 +186,7 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
request.type = package
request.name = name
request.storage_channel = repo
request.storage_url = gpts_path
request.storage_url = str(gpts_path)
request.author = ",".join(base_package.authors)
request.email = ",".join(base_package.authors)
@@ -174,7 +194,7 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
request.installed = 0
request.version = base_package.version
request.description = base_package.description
self.dao.create(request)
self.create(request)
except Exception as e:
raise ValueError(f"Update Agent Hub Db Info Faild!{str(e)}")

19
dbgpt/serve/dbgpts/my/api/endpoints.py
View File

@@ -1,3 +1,4 @@
import logging
from functools import cache
from typing import List, Optional
@@ -18,6 +19,8 @@ router = APIRouter()
global_system_app: Optional[SystemApp] = None
logger = logging.getLogger(__name__)
def get_service() -> Service:
"""Get the service instance"""
@@ -171,6 +174,22 @@ async def query_page(
return Result.succ(service.get_list_by_page(request, page, page_size))
@router.post("uninstall", response_model=Result[str])
async def agent_uninstall(
name: str,
type=str,
user: Optional[str] = None,
service: Service = Depends(get_service),
):
logger.info(f"dbgpts uninstall:{name},{user}")
try:
service.uninstall_gpts(name=name, type=type, user_code=user)
return Result.succ(None)
except Exception as e:
logger.error("Plugin Uninstall Error!", e)
return Result.failed(code="E0022", msg=f"Plugin Uninstall Error {e}")
def init_endpoints(system_app: SystemApp) -> None:
"""Initialize the endpoints"""
global global_system_app

9
dbgpt/serve/dbgpts/my/api/schemas.py
View File

@@ -13,13 +13,12 @@ class ServeRequest(BaseModel):
user_code: Optional[str] = Field(None, description="My gpts user code")
user_name: Optional[str] = Field(None, description="My gpts user name")
sys_code: Optional[str] = Field(None, description="My gpts sys code")
name: str = Field(..., description="My gpts name")
file_name: str = Field(..., description="My gpts file name")
name: Optional[str] = Field(None, description="My gpts name")
file_name: Optional[str] = Field(None, description="My gpts file name")
type: Optional[str] = Field(None, description="My gpts type")
version: Optional[str] = Field(None, description="My gpts version")
use_count: Optional[int] = Field(None, description="My gpts use count")
succ_count: Optional[int] = Field(None, description="My gpts succ count")
gmt_created: Optional[str] = Field(None, description="My gpts install time")
model_config = ConfigDict(title=f"ServeRequest for {SERVE_APP_NAME_HUMP}")
@@ -28,4 +27,6 @@ class ServeRequest(BaseModel):
return model_to_dict(self, **kwargs)
ServerResponse = ServeRequest
class ServerResponse(ServeRequest):
gmt_created: Optional[str] = Field(None, description="Dbgpts create time")
gmt_modified: Optional[str] = Field(None, description="Dbgpts upload time")

20
dbgpt/serve/dbgpts/my/models/models.py
View File

@@ -29,6 +29,12 @@ class ServeEntity(Model):
)
sys_code = Column(String(128), index=True, nullable=True, comment="System code")
gmt_created = Column(DateTime, default=datetime.utcnow, comment="gpts install time")
gmt_modified = Column(
DateTime,
default=datetime.now,
onupdate=datetime.utcnow,
comment="Record update time",
)
UniqueConstraint("user_code", "name", name="uk_name")
@@ -51,7 +57,8 @@ class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
request_dict = (
request.to_dict() if isinstance(request, ServeRequest) else request
)
entity = ServeRequest(**request_dict)
entity = ServeEntity(**request_dict)
return entity
def to_request(self, entity: ServeEntity) -> ServeRequest:
@@ -74,7 +81,6 @@ class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
version=entity.version,
use_count=entity.use_count,
succ_count=entity.succ_count,
gmt_created=entity.gmt_created,
)
def to_response(self, entity: ServeEntity) -> ServerResponse:
@@ -86,4 +92,12 @@ class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
Returns:
RES: The response
"""
return self.to_request(entity)
gmt_created_str = entity.gmt_created.strftime("%Y-%m-%d %H:%M:%S")
gmt_modified_str = entity.gmt_modified.strftime("%Y-%m-%d %H:%M:%S")
request = self.to_request(entity)
return ServerResponse(
**request.to_dict(),
gmt_created=gmt_created_str,
gmt_modified=gmt_modified_str,
)

18
dbgpt/serve/dbgpts/my/service/service.py
View File

@@ -5,7 +5,12 @@ from dbgpt.component import BaseComponent, SystemApp
from dbgpt.serve.core import BaseService
from dbgpt.storage.metadata import BaseDao
from dbgpt.util.dbgpts.base import INSTALL_DIR
from dbgpt.util.dbgpts.repo import copy_and_install, install, uninstall
from dbgpt.util.dbgpts.repo import (
copy_and_install,
inner_copy_and_install,
install,
uninstall,
)
from dbgpt.util.pagination_utils import PaginationResult
from ..api.schemas import ServeRequest, ServerResponse
@@ -132,8 +137,11 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
# install(name, repo)
try:
copy_and_install(repo, name, dbgpt_path)
from pathlib import Path
inner_copy_and_install(repo, name, Path(dbgpt_path))
except Exception as e:
logger.exception(f"install_gpts failed!{str(e)}")
raise ValueError(f"Install dbgpts [{type}:{name}] Failed! {str(e)}", e)
from dbgpt.util.dbgpts.base import get_repo_path
@@ -147,11 +155,11 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
InstalledPackage(
name=name,
repo=repo,
root=get_repo_path(repo),
root=dbgpt_path,
package=type,
)
)
dbgpts_entity = self.dao.get_one(ServeRequest(name=name, type=type))
dbgpts_entity = self.get(ServeRequest(name=name, type=type))
if not dbgpts_entity:
request = ServeRequest()
@@ -160,7 +168,7 @@ class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
request.user_code = user_code
request.sys_code = sys_code
request.type = type
request.file_name = INSTALL_DIR / name
request.file_name = str(INSTALL_DIR / name)
request.version = base_package.version
return self.create(request)
else:

1
dbgpt/util/dbgpts/base.py
View File

@@ -15,7 +15,6 @@ DBGPTS_REPO_HOME = os.getenv("DBGPTS_REPO_HOME", str(DEFAULT_DBGPTS_DIR / "repos
DEFAULT_REPO_MAP = {
"eosphoros/dbgpts": "https://github.com/eosphoros-ai/dbgpts.git",
"fangyinc/dbgpts": "https://github.com/fangyinc/dbgpts.git",
}
DEFAULT_PACKAGES = ["agents", "apps", "operators", "workflow", "resources"]

42
dbgpt/util/dbgpts/repo.py
View File

@@ -1,4 +1,5 @@
import functools
import logging
import os
import shutil
import subprocess
@@ -24,6 +25,8 @@ cl = CliLogger()
_DEFAULT_REPO = "eosphoros/dbgpts"
logger = logging.getLogger(__name__)
@functools.cache
def list_repos() -> List[str]:
@@ -227,6 +230,45 @@ def uninstall(name: str):
cl.info(f"Uninstalling dbgpt '{name}'...")
def inner_copy_and_install(repo: str, name: str, package_path: Path):
if not package_path.exists():
raise ValueError(
f"The specified dbgpt '{name}' does not exist in the {repo} tap."
)
install_path = INSTALL_DIR / name
if install_path.exists():
logger.info(
f"The dbgpt '{name}' has already been installed"
f"({_print_path(install_path)})."
)
return True
try:
shutil.copytree(package_path, install_path)
logger.info(f"Installing dbgpts '{name}' from {repo}...")
os.chdir(install_path)
subprocess.run(["poetry", "build"], check=True)
wheel_files = list(install_path.glob("dist/*.whl"))
if not wheel_files:
logger.error(
"No wheel file found after building the package.",
)
raise ValueError("No wheel file found after building the package.")
# Install the wheel file using pip
wheel_file = wheel_files[0]
logger.info(
f"Installing dbgpts '{name}' wheel file {_print_path(wheel_file)}..."
)
subprocess.run(["pip", "install", str(wheel_file)], check=True)
_write_install_metadata(name, repo, install_path)
logger.info(f"Installed dbgpts at {_print_path(install_path)}.")
logger.info(f"dbgpts '{name}' installed successfully.")
except Exception as e:
if install_path.exists():
shutil.rmtree(install_path)
raise e
def copy_and_install(repo: str, name: str, package_path: Path):
if not package_path.exists():
cl.error(