From d5a2a0bf3b5d8bb85abbe161a2c78d4a6fc5ad5e Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Wed, 5 Mar 2025 13:33:03 +0800 Subject: [PATCH] refactor(dbgpts): Refactor dbgpts for 0.7.0 (#2397) --- packages/dbgpt-app/pyproject.toml | 1 + .../dbgpt-core/src/dbgpt/cli/cli_scripts.py | 2 + .../src/dbgpt/configs/model_config.py | 3 + .../src/dbgpt/core/interface/llm.py | 8 +- .../dbgpt/model/llm/llama_cpp/llama_cpp.py | 1 + .../model/llm/llama_cpp/llama_cpp_server.py | 1 + .../dbgpt/model/llm/llm_out/hf_chat_llm.py | 1 + .../src/dbgpt/model/llm/llm_out/vllm_llm.py | 1 + .../src/dbgpt/rag/embedding/embeddings.py | 2 +- .../dbgpt-core/src/dbgpt/util/dbgpts/cli.py | 108 ++++++- .../dbgpt-core/src/dbgpt/util/dbgpts/repo.py | 264 ++++++++++++++++-- .../src/dbgpt/util/dbgpts/template.py | 125 ++++++++- .../dbgpt-core/src/dbgpt/util/system_utils.py | 2 +- .../dbgpt_serve/flow/service/share_utils.py | 52 ++-- uv.lock | 2 + 15 files changed, 504 insertions(+), 69 deletions(-) diff --git a/packages/dbgpt-app/pyproject.toml b/packages/dbgpt-app/pyproject.toml index 3d5812473..19f49f960 100644 --- a/packages/dbgpt-app/pyproject.toml +++ b/packages/dbgpt-app/pyproject.toml @@ -43,6 +43,7 @@ base = [ ] dbgpts = [ # For build dbgpts apps, we will be removed in the future. + "build", "poetry" ] diff --git a/packages/dbgpt-core/src/dbgpt/cli/cli_scripts.py b/packages/dbgpt-core/src/dbgpt/cli/cli_scripts.py index f92463d6b..d66e9f551 100644 --- a/packages/dbgpt-core/src/dbgpt/cli/cli_scripts.py +++ b/packages/dbgpt-core/src/dbgpt/cli/cli_scripts.py @@ -187,6 +187,7 @@ try: list_installed_apps, list_repos, new_dbgpts, + reinstall, remove_repo, update_repo, ) @@ -200,6 +201,7 @@ try: add_command_alias(update_repo, name="update", parent_group=repo) add_command_alias(app_install, name="install", parent_group=app) add_command_alias(app_uninstall, name="uninstall", parent_group=app) + add_command_alias(reinstall, name="reinstall", parent_group=app) add_command_alias(app_list_remote, name="list-remote", parent_group=app) add_command_alias(list_installed_apps, name="list", parent_group=app) add_command_alias(new_dbgpts, name="app", parent_group=new) diff --git a/packages/dbgpt-core/src/dbgpt/configs/model_config.py b/packages/dbgpt-core/src/dbgpt/configs/model_config.py index 2354d09f8..093efa754 100644 --- a/packages/dbgpt-core/src/dbgpt/configs/model_config.py +++ b/packages/dbgpt-core/src/dbgpt/configs/model_config.py @@ -57,6 +57,9 @@ def get_device() -> str: except ModuleNotFoundError: return "cpu" + except Exception: + return "cpu" + LLM_MODEL_CONFIG = { "flan-t5-base": os.path.join(MODEL_PATH, "flan-t5-base"), diff --git a/packages/dbgpt-core/src/dbgpt/core/interface/llm.py b/packages/dbgpt-core/src/dbgpt/core/interface/llm.py index f3112ce8b..1bf97d9cf 100644 --- a/packages/dbgpt-core/src/dbgpt/core/interface/llm.py +++ b/packages/dbgpt-core/src/dbgpt/core/interface/llm.py @@ -447,6 +447,7 @@ class ModelOutput: error_code: int = 0, usage: Optional[Dict[str, Any]] = None, finish_reason: Optional[str] = None, + is_reasoning_model: bool = False, ) -> "ModelOutput": if thinking and text: # Has thinking and text @@ -460,7 +461,12 @@ class ModelOutput: content = MediaContent.build_text(text) else: # Build a empty thinking content - content = MediaContent.build_thinking(thinking) + # Handle empty data + content = ( + MediaContent.build_thinking(thinking) + if is_reasoning_model + else MediaContent.build_text("") + ) return cls( error_code=error_code, content=content, diff --git a/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp.py b/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp.py index ffb83ae7f..1cfa86e65 100644 --- a/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp.py +++ b/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp.py @@ -164,4 +164,5 @@ class LlamaCppModel: error_code=0, usage=usage, finish_reason=finish_reason, + is_reasoning_model=is_reasoning_model, ) diff --git a/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp_server.py b/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp_server.py index b54ec6b4a..dcf985d86 100644 --- a/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp_server.py +++ b/packages/dbgpt-core/src/dbgpt/model/llm/llama_cpp/llama_cpp_server.py @@ -110,6 +110,7 @@ def chat_generate_stream( error_code=0, finish_reason=finish_reason, usage=r.usage, + is_reasoning_model=is_reasoning_model, ) diff --git a/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/hf_chat_llm.py b/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/hf_chat_llm.py index 0399f9446..ded02ac42 100644 --- a/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/hf_chat_llm.py +++ b/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/hf_chat_llm.py @@ -93,4 +93,5 @@ def huggingface_chat_generate_stream( msg.reasoning_content, error_code=0, usage=usage, + is_reasoning_model=is_reasoning_model, ) diff --git a/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/vllm_llm.py b/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/vllm_llm.py index 1395c7a65..05e292625 100644 --- a/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/vllm_llm.py +++ b/packages/dbgpt-core/src/dbgpt/model/llm/llm_out/vllm_llm.py @@ -117,4 +117,5 @@ async def generate_stream( error_code=0, usage=usage, finish_reason=finish_reason, + is_reasoning_model=is_reasoning_model, ) diff --git a/packages/dbgpt-core/src/dbgpt/rag/embedding/embeddings.py b/packages/dbgpt-core/src/dbgpt/rag/embedding/embeddings.py index 295256796..4ab13d242 100644 --- a/packages/dbgpt-core/src/dbgpt/rag/embedding/embeddings.py +++ b/packages/dbgpt-core/src/dbgpt/rag/embedding/embeddings.py @@ -38,7 +38,7 @@ try: from transformers.utils import is_torch_npu_available transformers.is_torch_npu_available = is_torch_npu_available -except ImportError: +except Exception: pass diff --git a/packages/dbgpt-core/src/dbgpt/util/dbgpts/cli.py b/packages/dbgpt-core/src/dbgpt/util/dbgpts/cli.py index 380450f70..6ad2bc9e1 100644 --- a/packages/dbgpt-core/src/dbgpt/util/dbgpts/cli.py +++ b/packages/dbgpt-core/src/dbgpt/util/dbgpts/cli.py @@ -1,4 +1,5 @@ import functools +import shutil import subprocess from pathlib import Path @@ -10,23 +11,95 @@ from .base import DEFAULT_PACKAGE_TYPES cl = CliLogger() -def check_poetry_installed(): +def check_build_tools_installed(): + """Check if any supported build tools are installed (uv, poetry, build, or + setuptools) + + Warns if uv is not installed but does not exit. + Only exits if no build tools are available. + """ + tools_available = [] + + # Check for uv (preferred tool) + if shutil.which("uv"): + try: + subprocess.run( + ["uv", "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + tools_available.append("uv") + except subprocess.CalledProcessError: + pass + + # Check for poetry + if shutil.which("poetry"): + try: + subprocess.run( + ["poetry", "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + tools_available.append("poetry") + except subprocess.CalledProcessError: + pass + + # Check for build package try: - # Check if poetry is installed - subprocess.run( - ["poetry", "--version"], + # Check if python and build are available + result = subprocess.run( + ["python", "-c", "import build; print('yes')"], check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + capture_output=True, + text=True, ) + if "yes" in result.stdout: + tools_available.append("build") except (subprocess.CalledProcessError, FileNotFoundError): - cl.error("Poetry is not installed. Please install Poetry to proceed.") - # Exit with error + pass + + # Check for python with setuptools + try: + # Check if python and setuptools are available for basic package building + result = subprocess.run( + ["python", "-c", "import setuptools; print('yes')"], + check=True, + capture_output=True, + text=True, + ) + if "yes" in result.stdout: + tools_available.append("setuptools") + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + # Provide appropriate feedback based on available tools + if not tools_available: + cl.error("No build tools found. Please install one of the following:") + cl.error(" - uv: A fast Python package installer and resolver (recommended)") + cl.error(" - poetry: A Python package and dependency manager") + cl.error(" - build: A PEP 517 compatible Python package builder") + cl.error(" - setuptools: A classic Python package build system") cl.error( - "Visit https://python-poetry.org/docs/#installation for installation " - "instructions.", + "For uv: https://github.com/astral-sh/uv\n" + "For poetry: https://python-poetry.org/docs/#installation\n" + "For build: pip install build\n" + "For setuptools: pip install setuptools", exit_code=1, ) + elif "uv" not in tools_available: + cl.warning( + "uv is not installed. We recommend using uv for better performance." + " Install with: 'pip install uv' or visit https://github.com/astral-sh/uv" + ) + if "build" not in tools_available: + cl.warning( + "build package is not installed. For better compatibility without uv," + " we recommend installing it with: 'pip install build'" + ) + + return tools_available def add_tap_options(func): @@ -77,7 +150,7 @@ def install(repo: str | None, update: bool, names: list[str]): """Install your dbgpts(operators,agents,workflows or apps)""" from .repo import _install_default_repos_if_no_repos, install - check_poetry_installed() + check_build_tools_installed() _install_default_repos_if_no_repos() for name in names: install(name, repo, with_update=update) @@ -93,6 +166,17 @@ def uninstall(names: list[str]): uninstall(name) +@click.command(name="reinstall") +@add_add_common_options +@click.argument("names", type=str, nargs=-1) +def reinstall(repo: str | None, update: bool, names: list[str]): + """Reinstall your dbgpts(operators,agents,workflows or apps)""" + from .repo import reinstall + + for name in names: + reinstall(name, repo, with_update=update) + + @click.command(name="list-remote") @add_add_common_options def list_all_apps( @@ -253,7 +337,7 @@ def new_dbgpts( type=click.Path(exists=True, file_okay=False, dir_okay=True), ) - check_poetry_installed() + check_build_tools_installed() from .template import create_template create_template(name, label, description, type, definition_type, directory) diff --git a/packages/dbgpt-core/src/dbgpt/util/dbgpts/repo.py b/packages/dbgpt-core/src/dbgpt/util/dbgpts/repo.py index 5be565b02..474b60d59 100644 --- a/packages/dbgpt-core/src/dbgpt/util/dbgpts/repo.py +++ b/packages/dbgpt-core/src/dbgpt/util/dbgpts/repo.py @@ -1,8 +1,11 @@ import functools +import importlib.metadata +import importlib.util import logging import os import shutil import subprocess +import sys from pathlib import Path from typing import List, Tuple @@ -28,6 +31,183 @@ _DEFAULT_REPO = "eosphoros/dbgpts" logger = logging.getLogger(__name__) +def _is_poetry_project(package_path: Path) -> bool: + """Check if the project uses poetry build system by examining pyproject.toml + + Args: + package_path (Path): Path to the package directory + + Returns: + bool: True if the project uses poetry, False otherwise + """ + pyproject_path = package_path / "pyproject.toml" + if not pyproject_path.exists(): + return False + + try: + import tomli # Python 3.11+ uses tomllib from stdlib + except ImportError: + try: + import tomlkit as tomli + except ImportError: + import tomli + + with open(pyproject_path, "rb") as f: + pyproject_data = tomli.load(f) + + # Check if poetry is defined in build-system + build_system = pyproject_data.get("build-system", {}) + build_backend = build_system.get("build-backend", "") + requires = build_system.get("requires", []) + + return ( + "poetry" in build_backend.lower() + or any("poetry" in req.lower() for req in requires) + or "tool" in pyproject_data + and "poetry" in pyproject_data.get("tool", {}) + ) + + +def _is_module_installed(module_name: str) -> bool: + """Check if a Python module is installed + + Args: + module_name (str): Module name to check + + Returns: + bool: True if installed, False otherwise + """ + try: + importlib.metadata.distribution(module_name) + return True + except importlib.metadata.PackageNotFoundError: + return False + + +def _build_package( + install_path: Path, is_poetry: bool, log_func=logger.info, error_func=logger.error +) -> tuple[bool, str]: + """Build package using poetry or build module + + Args: + install_path (Path): Path to the package + is_poetry (bool): Whether to use poetry for build + log_func: Function to use for logging info + error_func: Function to use for logging errors + + Returns: + tuple: (success, error_message) + """ + python_executable = sys.executable + os.chdir(install_path) + + if is_poetry and shutil.which("poetry"): + log_func("Building with poetry...") + process = subprocess.Popen( + ["poetry", "build"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + else: + # Check if build module is available + if _is_module_installed("build"): + log_func("Building with python -m build...") + process = subprocess.Popen( + [python_executable, "-m", "build", "--wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + # Fallback to setuptools directly if available + if (install_path / "setup.py").exists(): + log_func("Building with setuptools directly...") + process = subprocess.Popen( + [python_executable, "setup.py", "bdist_wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + error_msg = ( + "No suitable build method found. Please install build: " + "pip install build" + ) + error_func(error_msg) + return False, error_msg + + out, err = process.communicate() + log_func(f"Build output: {out.decode('utf-8', errors='replace')}") + if err: + error_msg = f"Build warnings/errors: {err.decode('utf-8', errors='replace')}" + log_func(error_msg) + + return process.returncode == 0, "" if process.returncode == 0 else error_msg + + +def _install_wheel( + wheel_path: str, log_func=logger.info, error_func=logger.error +) -> tuple[bool, str]: + """Install wheel file using the best available method + + Args: + wheel_path (str): Path to the wheel file + log_func: Function to use for logging info + error_func: Function to use for logging errors + + Returns: + tuple: (success, error_message) + """ + python_executable = sys.executable + + # Try uv first if available (faster) + if shutil.which("uv"): + log_func(f"Installing wheel with uv: {wheel_path}") + process = subprocess.Popen( + ["uv", "pip", "install", wheel_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = process.communicate() + if process.returncode == 0: + return True, "" + log_func(f"uv install failed: {err.decode('utf-8', errors='replace')}") + # Fall through to pip if uv fails + + # Try pip + if shutil.which("pip") or _is_module_installed("pip"): + log_func(f"Installing wheel with pip: {wheel_path}") + process = subprocess.Popen( + [python_executable, "-m", "pip", "install", wheel_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + out, err = process.communicate() + if process.returncode == 0: + return True, "" + log_func(f"Pip install failed: {err.decode('utf-8', errors='replace')}") + + # Alternative: Try using importlib directly + try: + import site + import zipfile + + log_func(f"Attempting manual wheel installation for {wheel_path}") + + # Get the site-packages directory + site_packages = site.getsitepackages()[0] + + # Extract the wheel contents to site-packages + with zipfile.ZipFile(wheel_path, "r") as wheel_zip: + # Extract only the Python modules (*.py, *.so, etc.) + for file in wheel_zip.namelist(): + if ".dist-info/" not in file and file.endswith((".py", ".so", ".pyd")): + wheel_zip.extract(file, site_packages) + + log_func(f"Manual installation to {site_packages} completed") + return True, "" + except Exception as e: + error_msg = f"Manual installation failed: {str(e)}" + error_func(error_msg) + return False, error_msg + + @functools.cache def list_repos() -> List[str]: """List all repos @@ -234,6 +414,20 @@ def uninstall(name: str): cl.info(f"Uninstalling dbgpt '{name}'...") +def reinstall( + name: str, + repo: str | None = None, + with_update: bool = True, +): + """Reinstall the specified dbgpt + + Args: + name (str): The name of the dbgpt + """ + uninstall(name) + install(name, repo, with_update=with_update) + + def inner_uninstall(name: str): """Uninstall the specified dbgpt @@ -271,35 +465,34 @@ def inner_copy_and_install(repo: str, name: str, package_path: Path): try: shutil.copytree(package_path, install_path) logger.info(f"Installing dbgpts '{name}' from {repo}...") - os.chdir(install_path) - process = subprocess.Popen( - ["poetry", "build"], stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - out, err = process.communicate() - logger.info(f"{out},{err}") + + # Check if it's a poetry project + is_poetry = _is_poetry_project(install_path) + + # Build the package + build_success = _build_package(install_path, is_poetry) + if not build_success: + raise ValueError("Failed to build the package") wheel_files = list(install_path.glob("dist/*.whl")) if not wheel_files: - logger.error( - "No wheel file found after building the package.", - ) + logger.error("No wheel file found after building the package.") raise ValueError("No wheel file found after building the package.") - # Install the wheel file using pip + + # Install the wheel file wheel_file = wheel_files[0] logger.info( f"Installing dbgpts '{name}' wheel file {_print_path(wheel_file)}..." ) - # subprocess.run(["pip", "install", str(wheel_file)], check=True) - process = subprocess.Popen( - ["pip", "install", str(wheel_file)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - out, err = process.communicate() - logger.info(f"{out},{err}") + + install_success = _install_wheel(str(wheel_file)) + if not install_success: + raise ValueError(f"Failed to install wheel file: {wheel_file}") + _write_install_metadata(name, repo, install_path) logger.info(f"Installed dbgpts at {_print_path(install_path)}.") logger.info(f"dbgpts '{name}' installed successfully.") + return True except Exception as e: if install_path.exists(): shutil.rmtree(install_path) @@ -307,6 +500,15 @@ def inner_copy_and_install(repo: str, name: str, package_path: Path): def copy_and_install(repo: str, name: str, package_path: Path): + """Install the specified dbgpt from a repository + + This function reuses the inner implementation but provides CLI feedback + + Args: + repo (str): The name of the repo + name (str): The name of the dbgpt + package_path (Path): The path to the package + """ if not package_path.exists(): cl.error( f"The specified dbgpt '{name}' does not exist in the {repo} tap.", @@ -322,15 +524,33 @@ def copy_and_install(repo: str, name: str, package_path: Path): try: shutil.copytree(package_path, install_path) cl.info(f"Installing dbgpts '{name}' from {repo}...") - os.chdir(install_path) - subprocess.run(["poetry", "build"], check=True) + + # Check if it's a poetry project + is_poetry = _is_poetry_project(install_path) + + # Build the package using the shared implementation but with CLI logging + build_success, build_error = _build_package( + install_path, is_poetry, log_func=cl.info, error_func=cl.error + ) + + if not build_success: + cl.error(f"Failed to build the package: {build_error}", exit_code=1) + wheel_files = list(install_path.glob("dist/*.whl")) if not wheel_files: cl.error("No wheel file found after building the package.", exit_code=1) - # Install the wheel file using pip + + # Install the wheel file using the shared implementation but with CLI logging wheel_file = wheel_files[0] cl.info(f"Installing dbgpts '{name}' wheel file {_print_path(wheel_file)}...") - subprocess.run(["pip", "install", str(wheel_file)], check=True) + + install_success, install_error = _install_wheel( + str(wheel_file), log_func=cl.info, error_func=cl.error + ) + + if not install_success: + cl.error(f"Failed to install wheel file: {install_error}", exit_code=1) + _write_install_metadata(name, repo, install_path) cl.success(f"Installed dbgpts at {_print_path(install_path)}.") cl.success(f"dbgpts '{name}' installed successfully.") diff --git a/packages/dbgpt-core/src/dbgpt/util/dbgpts/template.py b/packages/dbgpt-core/src/dbgpt/util/dbgpts/template.py index bc64c7d22..643cdc806 100644 --- a/packages/dbgpt-core/src/dbgpt/util/dbgpts/template.py +++ b/packages/dbgpt-core/src/dbgpt/util/dbgpts/template.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess from pathlib import Path @@ -90,7 +91,9 @@ def _create_flow_template( if definition_type == "json": json_dict["json_config"] = {"file_path": "definition/flow_definition.json"} - _create_poetry_project(working_directory, name) + # Create project structure + _create_project_structure(working_directory, name, base_metadata.get("description")) + _write_dbgpts_toml(working_directory, name, json_dict) _write_manifest_file(working_directory, name, mod_name) @@ -120,8 +123,9 @@ def _create_operator_template( f"Unsupported definition type: {definition_type} for dbgpts type: " f"{dbgpts_type}" ) + # Create project structure + _create_project_structure(working_directory, name, base_metadata.get("description")) - _create_poetry_project(working_directory, name) _write_dbgpts_toml(working_directory, name, json_dict) _write_operator_init_file(working_directory, name, mod_name) _write_manifest_file(working_directory, name, mod_name) @@ -146,7 +150,9 @@ def _create_agent_template( f"{dbgpts_type}" ) - _create_poetry_project(working_directory, name) + # Create project structure + _create_project_structure(working_directory, name, base_metadata.get("description")) + _write_dbgpts_toml(working_directory, name, json_dict) _write_agent_init_file(working_directory, name, mod_name) _write_manifest_file(working_directory, name, mod_name) @@ -171,17 +177,122 @@ def _create_resource_template( f"{dbgpts_type}" ) - _create_poetry_project(working_directory, name) + # Create project structure + _create_project_structure(working_directory, name, base_metadata.get("description")) + _write_dbgpts_toml(working_directory, name, json_dict) _write_resource_init_file(working_directory, name, mod_name) _write_manifest_file(working_directory, name, mod_name) -def _create_poetry_project(working_directory: str, name: str): - """Create a new poetry project""" +def _create_project_structure( + working_directory: str, name: str, description: str = None +): + """Create a new project using uv, poetry or manual file creation + Args: + working_directory (str): Directory to create the project in + name (str): Name of the project + description (str, optional): Project description + + Returns: + bool: True if project created successfully + """ os.chdir(working_directory) - subprocess.run(["poetry", "new", name, "-n"], check=True) + + # Try uv first + if shutil.which("uv"): + try: + cmd = ["uv", "init", "--no-workspace"] + + if description: + cmd.extend(["--description", description]) + + cmd.append(name) + + subprocess.run(cmd, check=True) + return True + except subprocess.CalledProcessError: + click.echo("Warning: Failed to create project with uv, trying poetry...") + + # Try poetry next + if shutil.which("poetry"): + try: + subprocess.run(["poetry", "new", name, "-n"], check=True) + + # If description provided, update pyproject.toml + if description: + pyproject_path = Path(working_directory) / name / "pyproject.toml" + if pyproject_path.exists(): + _update_pyproject_description(pyproject_path, description) + + return True + except subprocess.CalledProcessError: + click.echo( + "Warning: Failed to create project with poetry, creating files " + "manually..." + ) + + # Manual creation as fallback + project_dir = Path(working_directory) / name + project_dir.mkdir(parents=True, exist_ok=True) + + # Create basic project structure + _create_manual_project_structure(project_dir, name, description) + + return True + + +def _update_pyproject_description(pyproject_path, description): + """Update description in pyproject.toml file""" + try: + import tomlkit + + with open(pyproject_path, "r") as f: + pyproject = tomlkit.parse(f.read()) + + if "tool" in pyproject and "poetry" in pyproject["tool"]: + pyproject["tool"]["poetry"]["description"] = description + elif "project" in pyproject: + pyproject["project"]["description"] = description + + with open(pyproject_path, "w") as f: + f.write(tomlkit.dumps(pyproject)) + except Exception as e: + click.echo(f"Warning: Failed to update description in pyproject.toml: {e}") + + +def _create_manual_project_structure(project_dir, name, description=None): + """Create manual project structure with necessary files""" + mod_name = name.replace("-", "_") + + # Create module directory + module_dir = project_dir / mod_name + module_dir.mkdir(parents=True, exist_ok=True) + + # Create __init__.py + with open(module_dir / "__init__.py", "w") as f: + f.write(f'"""Main module for {name}."""\n\n') + + # Create pyproject.toml + pyproject_content = f"""[project] +name = "{name}" +version = "0.1.0" +description = "{description or f"A {name} package"}" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" +""" + with open(project_dir / "pyproject.toml", "w") as f: + f.write(pyproject_content) + + # Create README.md + with open(project_dir / "README.md", "w") as f: + f.write(f"# {name}\n\n{description or f'A {name} package'}\n") def _write_dbgpts_toml(working_directory: str, name: str, json_data: dict): diff --git a/packages/dbgpt-core/src/dbgpt/util/system_utils.py b/packages/dbgpt-core/src/dbgpt/util/system_utils.py index d40a02ae7..721623a3c 100644 --- a/packages/dbgpt-core/src/dbgpt/util/system_utils.py +++ b/packages/dbgpt-core/src/dbgpt/util/system_utils.py @@ -120,7 +120,7 @@ def get_device_info() -> Tuple[str, str, str, int, str]: device_count = torch.cuda.device_count() elif torch.backends.mps.is_available(): device = "mps" - except ModuleNotFoundError: + except Exception: pass if not device_version: diff --git a/packages/dbgpt-serve/src/dbgpt_serve/flow/service/share_utils.py b/packages/dbgpt-serve/src/dbgpt_serve/flow/service/share_utils.py index 8405d9dd6..ef8629464 100644 --- a/packages/dbgpt-serve/src/dbgpt_serve/flow/service/share_utils.py +++ b/packages/dbgpt-serve/src/dbgpt_serve/flow/service/share_utils.py @@ -20,80 +20,82 @@ def _generate_dbgpts_zip(package_name: str, flow: ServeRequest) -> io.BytesIO: flow_label = flow.label flow_description = flow.description dag_json = json.dumps(flow.flow_data.dict(), indent=4, ensure_ascii=False) + with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file: + # Create MANIFEST.in manifest = f"include dbgpts.toml\ninclude {flow_name}/definition/*.json" - readme = f"# {flow_label}\n\n{flow_description}" zip_file.writestr(f"{package_name}/MANIFEST.in", manifest) + + # Create README.md + readme = f"# {flow_label}\n\n{flow_description}" zip_file.writestr(f"{package_name}/README.md", readme) + + # Create module __init__.py zip_file.writestr( f"{package_name}/{flow_name}/__init__.py", "", ) + + # Create flow definition json zip_file.writestr( f"{package_name}/{flow_name}/definition/flow_definition.json", dag_json, ) + + # Create dbgpts.toml dbgpts_toml = tomlkit.document() # Add flow information dbgpts_flow_toml = tomlkit.document() - dbgpts_flow_toml.add("label", "Simple Streaming Chat") - name_with_comment = tomlkit.string("awel_flow_simple_streaming_chat") + dbgpts_flow_toml.add("label", flow_label) + name_with_comment = tomlkit.string(flow_name) name_with_comment.comment("A unique name for all dbgpts") dbgpts_flow_toml.add("name", name_with_comment) - dbgpts_flow_toml.add("version", "0.1.0") dbgpts_flow_toml.add( "description", flow_description, ) dbgpts_flow_toml.add("authors", []) - definition_type_with_comment = tomlkit.string("json") definition_type_with_comment.comment("How to define the flow, python or json") dbgpts_flow_toml.add("definition_type", definition_type_with_comment) - dbgpts_toml.add("flow", dbgpts_flow_toml) # Add python and json config python_config = tomlkit.table() dbgpts_toml.add("python_config", python_config) - json_config = tomlkit.table() json_config.add("file_path", "definition/flow_definition.json") json_config.comment("Json config") - dbgpts_toml.add("json_config", json_config) - # Transform to string + # Transform dbgpts.toml to string toml_string = tomlkit.dumps(dbgpts_toml) zip_file.writestr(f"{package_name}/dbgpts.toml", toml_string) + # Create pyproject.toml (uv style) pyproject_toml = tomlkit.document() - # Add [tool.poetry] section - tool_poetry_toml = tomlkit.table() - tool_poetry_toml.add("name", package_name) - tool_poetry_toml.add("version", "0.1.0") - tool_poetry_toml.add("description", "A dbgpts package") - tool_poetry_toml.add("authors", []) - tool_poetry_toml.add("readme", "README.md") - pyproject_toml["tool"] = tomlkit.table() - pyproject_toml["tool"]["poetry"] = tool_poetry_toml - - # Add [tool.poetry.dependencies] section - dependencies = tomlkit.table() - dependencies.add("python", "^3.10") - pyproject_toml["tool"]["poetry"]["dependencies"] = dependencies + # Add [project] section (modern PEP 621 format used by uv) + project_section = tomlkit.table() + project_section.add("name", package_name) + project_section.add("version", "0.1.0") + project_section.add("description", flow_description or "A dbgpts package") + project_section.add("readme", "README.md") + project_section.add("requires-python", ">=3.8") + project_section.add("dependencies", []) + pyproject_toml["project"] = project_section # Add [build-system] section build_system = tomlkit.table() - build_system.add("requires", ["poetry-core"]) - build_system.add("build-backend", "poetry.core.masonry.api") + build_system.add("requires", ["setuptools>=61.0"]) + build_system.add("build-backend", "setuptools.build_meta") pyproject_toml["build-system"] = build_system # Transform to string pyproject_toml_string = tomlkit.dumps(pyproject_toml) zip_file.writestr(f"{package_name}/pyproject.toml", pyproject_toml_string) + zip_buffer.seek(0) return zip_buffer diff --git a/uv.lock b/uv.lock index c48ed52d1..106913565 100644 --- a/uv.lock +++ b/uv.lock @@ -1799,6 +1799,7 @@ cache = [ { name = "rocksdict" }, ] dbgpts = [ + { name = "build" }, { name = "poetry" }, ] observability = [ @@ -1810,6 +1811,7 @@ observability = [ [package.metadata] requires-dist = [ { name = "aiofiles" }, + { name = "build", marker = "extra == 'dbgpts'" }, { name = "dbgpt", editable = "packages/dbgpt-core" }, { name = "dbgpt", extras = ["client", "cli", "agent", "simple-framework", "framework", "code"], marker = "extra == 'base'", editable = "packages/dbgpt-core" }, { name = "dbgpt-acc-auto", virtual = "packages/dbgpt-accelerator/dbgpt-acc-auto" },