refactor: Refactor storage and new serve template (#947)

dbgpt/serve/utils/_template_files/default_serve_template/__init__.py

@@ -0,0 +1,2 @@
+# This is an auto-generated __init__.py file
+# generated by `dbgpt new serve {__template_app_name__}`

dbgpt/serve/utils/_template_files/default_serve_template/api/__init__.py

@@ -0,0 +1,2 @@
+# This is an auto-generated __init__.py file
+# generated by `dbgpt new serve {__template_app_name__}`

dbgpt/serve/utils/_template_files/default_serve_template/api/endpoints.py

@@ -0,0 +1,98 @@
+from typing import Optional, List
+from fastapi import APIRouter, Depends, Query
+
+from dbgpt.component import SystemApp
+from dbgpt.serve.core import Result
+from dbgpt.util import PaginationResult
+from .schemas import ServeRequest, ServerResponse
+from ..service.service import Service
+from ..config import APP_NAME, SERVE_APP_NAME, ServeConfig, SERVE_SERVICE_COMPONENT_NAME
+
+router = APIRouter()
+
+# Add your API endpoints here
+
+global_system_app: Optional[SystemApp] = None
+
+
+def get_service() -> Service:
+    """Get the service instance"""
+    return global_system_app.get_component(SERVE_SERVICE_COMPONENT_NAME, Service)
+
+
+@router.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {"status": "ok"}
+
+
+@router.post("/", response_model=Result[ServerResponse])
+async def create(
+    request: ServeRequest, service: Service = Depends(get_service)
+) -> Result[ServerResponse]:
+    """Create a new {__template_app_name__hump__} entity
+
+    Args:
+        request (ServeRequest): The request
+        service (Service): The service
+    Returns:
+        ServerResponse: The response
+    """
+    return Result.succ(service.create(request))
+
+
+@router.put("/", response_model=Result[ServerResponse])
+async def update(
+    request: ServeRequest, service: Service = Depends(get_service)
+) -> Result[ServerResponse]:
+    """Update a {__template_app_name__hump__} entity
+
+    Args:
+        request (ServeRequest): The request
+        service (Service): The service
+    Returns:
+        ServerResponse: The response
+    """
+    return Result.succ(service.update(request))
+
+
+@router.post("/query", response_model=Result[ServerResponse])
+async def query(
+    request: ServeRequest, service: Service = Depends(get_service)
+) -> Result[ServerResponse]:
+    """Query {__template_app_name__hump__} entities
+
+    Args:
+        request (ServeRequest): The request
+        service (Service): The service
+    Returns:
+        ServerResponse: The response
+    """
+    return Result.succ(service.get(request))
+
+
+@router.post("/query_page", response_model=Result[PaginationResult[ServerResponse]])
+async def query_page(
+    request: ServeRequest,
+    page: Optional[int] = Query(default=1, description="current page"),
+    page_size: Optional[int] = Query(default=20, description="page size"),
+    service: Service = Depends(get_service),
+) -> Result[PaginationResult[ServerResponse]]:
+    """Query {__template_app_name__hump__} entities
+
+    Args:
+        request (ServeRequest): The request
+        page (int): The page number
+        page_size (int): The page size
+        service (Service): The service
+    Returns:
+        ServerResponse: The response
+    """
+    return Result.succ(service.get_list_by_page(request, page, page_size))
+
+
+def init_endpoints(system_app: SystemApp) -> None:
+    """Initialize the endpoints"""
+    global global_system_app
+    system_app.register(Service)
+    global_system_app = system_app

dbgpt/serve/utils/_template_files/default_serve_template/api/schemas.py

@@ -0,0 +1,14 @@
+# Define your Pydantic schemas here
+from dbgpt._private.pydantic import BaseModel, Field
+
+
+class ServeRequest(BaseModel):
+    """{__template_app_name__hump__} request model"""
+
+    # TODO define your own fields here
+
+
+class ServerResponse(BaseModel):
+    """{__template_app_name__hump__} response model"""
+
+    # TODO define your own fields here

dbgpt/serve/utils/_template_files/default_serve_template/config.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+from dbgpt.serve.core import BaseServeConfig
+
+
+APP_NAME = "{__template_app_name__all_lower__}"
+SERVE_APP_NAME = "dbgpt_serve_{__template_app_name__all_lower__}"
+SERVE_APP_NAME_HUMP = "dbgpt_serve_{__template_app_name__hump__}"
+SERVE_CONFIG_KEY_PREFIX = "dbgpt.serve.{__template_app_name__all_lower__}."
+SERVE_SERVICE_COMPONENT_NAME = f"{SERVE_APP_NAME}_service"
+# Database table name
+SERVER_APP_TABLE_NAME = "dbgpt_serve_{__template_app_name__all_lower__}"
+
+
+@dataclass
+class ServeConfig(BaseServeConfig):
+    """Parameters for the serve command"""
+
+    # TODO: add your own parameters here

dbgpt/serve/utils/_template_files/default_serve_template/dependencies.py

@@ -0,0 +1 @@
+# Define your dependencies here

dbgpt/serve/utils/_template_files/default_serve_template/models/__init__.py

@@ -0,0 +1,2 @@
+# This is an auto-generated __init__.py file
+# generated by `dbgpt new serve {__template_app_name__}`

dbgpt/serve/utils/_template_files/default_serve_template/models/models.py

@@ -0,0 +1,68 @@
+"""This is an auto-generated model file
+You can define your own models and DAOs here
+"""
+from typing import Union, Any, Dict
+from datetime import datetime
+from sqlalchemy import Column, Integer, String, Index, Text, DateTime
+from dbgpt.storage.metadata import Model, BaseDao, db
+from ..api.schemas import ServeRequest, ServerResponse
+from ..config import ServeConfig, SERVER_APP_TABLE_NAME
+
+
+class ServeEntity(Model):
+    __tablename__ = SERVER_APP_TABLE_NAME
+    id = Column(Integer, primary_key=True, comment="Auto increment id")
+
+    # TODO: define your own fields here
+
+    gmt_created = Column(DateTime, default=datetime.now, comment="Record creation time")
+    gmt_modified = Column(DateTime, default=datetime.now, comment="Record update time")
+
+    def __repr__(self):
+        return f"ServeEntity(id={self.id}, gmt_created='{self.gmt_created}', gmt_modified='{self.gmt_modified}')"
+
+
+class ServeDao(BaseDao[ServeEntity, ServeRequest, ServerResponse]):
+    """The DAO class for {__template_app_name__hump__}"""
+
+    def __init__(self, serve_config: ServeConfig):
+        super().__init__()
+        self._serve_config = serve_config
+
+    def from_request(self, request: Union[ServeRequest, Dict[str, Any]]) -> ServeEntity:
+        """Convert the request to an entity
+
+        Args:
+            request (Union[ServeRequest, Dict[str, Any]]): The request
+
+        Returns:
+            T: The entity
+        """
+        request_dict = request.dict() if isinstance(request, ServeRequest) else request
+        entity = ServeEntity(**request_dict)
+        # TODO implement your own logic here, transfer the request_dict to an entity
+        return entity
+
+    def to_request(self, entity: ServeEntity) -> ServeRequest:
+        """Convert the entity to a request
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            REQ: The request
+        """
+        # TODO implement your own logic here, transfer the entity to a request
+        return ServeRequest()
+
+    def to_response(self, entity: ServeEntity) -> ServerResponse:
+        """Convert the entity to a response
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            RES: The response
+        """
+        # TODO implement your own logic here, transfer the entity to a response
+        return ServerResponse()

dbgpt/serve/utils/_template_files/default_serve_template/serve.py

@@ -0,0 +1,38 @@
+from typing import List, Optional
+from dbgpt.component import BaseComponent, SystemApp
+
+from .api.endpoints import router, init_endpoints
+from .config import SERVE_APP_NAME, SERVE_APP_NAME_HUMP, APP_NAME
+
+
+class Serve(BaseComponent):
+    """Serve component for DB-GPT"""
+
+    name = SERVE_APP_NAME
+
+    def __init__(
+        self,
+        system_app: SystemApp,
+        api_prefix: Optional[str] = f"/api/v1/serve/{APP_NAME}",
+        tags: Optional[List[str]] = None,
+    ):
+        if tags is None:
+            tags = [SERVE_APP_NAME_HUMP]
+        self._system_app = None
+        self._api_prefix = api_prefix
+        self._tags = tags
+
+    def init_app(self, system_app: SystemApp):
+        self._system_app = system_app
+        self._system_app.app.include_router(
+            router, prefix=self._api_prefix, tags=self._tags
+        )
+        init_endpoints(self._system_app)
+
+    def before_start(self):
+        """Called before the start of the application.
+
+        You can do some initialization here.
+        """
+        # import your own module here to ensure the module is loaded before the application starts
+        from .models.models import ServeEntity

dbgpt/serve/utils/_template_files/default_serve_template/service/service.py

@@ -0,0 +1,116 @@
+from typing import Optional, List
+from dbgpt.component import BaseComponent, SystemApp
+from dbgpt.storage.metadata import BaseDao
+from dbgpt.util.pagination_utils import PaginationResult
+from dbgpt.serve.core import BaseService
+from ..models.models import ServeDao, ServeEntity
+from ..api.schemas import ServeRequest, ServerResponse
+from ..config import SERVE_SERVICE_COMPONENT_NAME, SERVE_CONFIG_KEY_PREFIX, ServeConfig
+
+
+class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]):
+    """The service class for {__template_app_name__hump__}"""
+
+    name = SERVE_SERVICE_COMPONENT_NAME
+
+    def __init__(self, system_app: SystemApp):
+        self._system_app = None
+        self._serve_config: ServeConfig = None
+        self._dao: ServeDao = None
+        super().__init__(system_app)
+
+    def init_app(self, system_app: SystemApp) -> None:
+        """Initialize the service
+
+        Args:
+            system_app (SystemApp): The system app
+        """
+        self._serve_config = ServeConfig.from_app_config(
+            system_app.config, SERVE_CONFIG_KEY_PREFIX
+        )
+        self._dao = ServeDao(self._serve_config)
+        self._system_app = system_app
+
+    @property
+    def dao(self) -> BaseDao[ServeEntity, ServeRequest, ServerResponse]:
+        """Returns the internal DAO."""
+        return self._dao
+
+    @property
+    def config(self) -> ServeConfig:
+        """Returns the internal ServeConfig."""
+        return self._serve_config
+
+    def update(self, request: ServeRequest) -> ServerResponse:
+        """Update a {__template_app_name__hump__} entity
+
+        Args:
+            request (ServeRequest): The request
+
+        Returns:
+            ServerResponse: The response
+        """
+        # TODO: implement your own logic here
+        # Build the query request from the request
+        query_request = {
+            # "id": request.id
+        }
+        return self.dao.update(query_request, update_request=request)
+
+    def get(self, request: ServeRequest) -> Optional[ServerResponse]:
+        """Get a {__template_app_name__hump__} entity
+
+        Args:
+            request (ServeRequest): The request
+
+        Returns:
+            ServerResponse: The response
+        """
+        # TODO: implement your own logic here
+        # Build the query request from the request
+        query_request = request
+        return self.dao.get_one(query_request)
+
+    def delete(self, request: ServeRequest) -> None:
+        """Delete a {__template_app_name__hump__} entity
+
+        Args:
+            request (ServeRequest): The request
+        """
+
+        # TODO: implement your own logic here
+        # Build the query request from the request
+        query_request = {
+            # "id": request.id
+        }
+        self.dao.delete(query_request)
+
+    def get_list(self, request: ServeRequest) -> List[ServerResponse]:
+        """Get a list of {__template_app_name__hump__} entities
+
+        Args:
+            request (ServeRequest): The request
+
+        Returns:
+            List[ServerResponse]: The response
+        """
+        # TODO: implement your own logic here
+        # Build the query request from the request
+        query_request = request
+        return self.dao.get_list(query_request)
+
+    def get_list_by_page(
+        self, request: ServeRequest, page: int, page_size: int
+    ) -> PaginationResult[ServerResponse]:
+        """Get a list of {__template_app_name__hump__} entities by page
+
+        Args:
+            request (ServeRequest): The request
+            page (int): The page number
+            page_size (int): The page size
+
+        Returns:
+            List[ServerResponse]: The response
+        """
+        query_request = request
+        return self.dao.get_list_page(query_request, page, page_size)

dbgpt/serve/utils/cli.py

@@ -0,0 +1,83 @@
+import os
+import click
+
+
+@click.command(name="serve")
+@click.option(
+    "-n",
+    "--name",
+    required=True,
+    type=str,
+    show_default=True,
+    help="The name of the serve module to create",
+)
+@click.option(
+    "-t",
+    "--template",
+    required=False,
+    type=str,
+    default="default_serve_template",
+    show_default=True,
+    help="The template to use to create the serve module",
+)
+def serve(name: str, template: str):
+    """Create a serve module structure with the given name."""
+    from dbgpt.configs.model_config import ROOT_PATH
+
+    base_path = os.path.join(ROOT_PATH, "dbgpt", "serve", name)
+    template_path = os.path.join(
+        ROOT_PATH, "dbgpt", "serve", "utils", "_template_files", template
+    )
+    if not os.path.exists(template_path):
+        raise ValueError(f"Template '{template}' not found")
+    if os.path.exists(base_path):
+        # TODO: backup the old serve module
+        click.confirm(
+            f"Serve module '{name}' already exists in {base_path}, do you want to overwrite it?",
+            abort=True,
+        )
+        import shutil
+
+        shutil.rmtree(base_path)
+
+    copy_template_files(template_path, base_path, name)
+    click.echo(f"Serve application '{name}' created successfully in {base_path}")
+
+
+def replace_template_variables(content: str, app_name: str):
+    """Replace the template variables in the given content with the given app name."""
+    template_values = {
+        "{__template_app_name__}": app_name,
+        "{__template_app_name__all_lower__}": app_name.lower(),
+        "{__template_app_name__hump__}": "".join(
+            part.capitalize() for part in app_name.split("_")
+        ),
+    }
+
+    for key in sorted(template_values, key=len, reverse=True):
+        content = content.replace(key, template_values[key])
+
+    return content
+
+
+def copy_template_files(src_dir: str, dst_dir: str, app_name: str):
+    for root, dirs, files in os.walk(src_dir):
+        relative_path = os.path.relpath(root, src_dir)
+        if relative_path == ".":
+            relative_path = ""
+
+        target_dir = os.path.join(dst_dir, relative_path)
+        os.makedirs(target_dir, exist_ok=True)
+
+        for file in files:
+            try:
+                with open(os.path.join(root, file), "r") as f:
+                    content = f.read()
+
+                content = replace_template_variables(content, app_name)
+
+                with open(os.path.join(target_dir, file), "w") as f:
+                    f.write(content)
+            except Exception as e:
+                click.echo(f"Error copying file {file} from {src_dir} to {dst_dir}")
+                raise e