feat: AWEL flow supports dynamic parameters (#1251)

dbgpt/app/scene/chat_knowledge/v1/chat.py

@@ -208,7 +208,9 @@ class ChatKnowledge(BaseChat):
                     }
                 )
         references_list = list(references_dict.values())
-        references_ele.set("references", json.dumps(references_list))
+        references_ele.set(
+            "references", json.dumps(references_list, ensure_ascii=False)
+        )
         html = ET.tostring(references_ele, encoding="utf-8")
         reference = html.decode("utf-8")
         return reference.replace("\\n", "")

dbgpt/core/awel/flow/__init__.py

@@ -3,11 +3,15 @@
 This module contains the classes and functions to build AWEL DAGs from serialized data.
 """
 
+from ..util.parameter_util import (  # noqa: F401
+    BaseDynamicOptions,
+    FunctionDynamicOptions,
+    OptionValue,
+)
 from .base import (  # noqa: F401
     IOField,
     OperatorCategory,
     OperatorType,
-    OptionValue,
     Parameter,
     ResourceCategory,
     ResourceMetadata,
@@ -29,4 +33,6 @@ __ALL__ = [
     "ResourceType",
     "OperatorType",
     "IOField",
+    "BaseDynamicOptions",
+    "FunctionDynamicOptions",
 ]

dbgpt/core/awel/flow/base.py

@@ -8,6 +8,7 @@ from enum import Enum
 from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast
 
 from dbgpt._private.pydantic import BaseModel, Field, ValidationError, root_validator
+from dbgpt.core.awel.util.parameter_util import BaseDynamicOptions, OptionValue
 from dbgpt.core.interface.serialization import Serializable
 
 from .exceptions import FlowMetadataException, FlowParameterMetadataException
@@ -205,18 +206,6 @@ class ResourceType(str, Enum):
     CLASS = "class"
 
 
-class OptionValue(Serializable, BaseModel):
-    """The option value of the parameter."""
-
-    label: str = Field(..., description="The label of the option")
-    name: str = Field(..., description="The name of the option")
-    value: Any = Field(..., description="The value of the option")
-
-    def to_dict(self) -> Dict:
-        """Convert current metadata to json dict."""
-        return self.dict()
-
-
 class ParameterType(str, Enum):
     """The type of the parameter."""
 
@@ -317,7 +306,7 @@ class Parameter(TypeMetadata, Serializable):
     description: Optional[str] = Field(
         None, description="The description of the parameter"
     )
-    options: Optional[List[OptionValue]] = Field(
+    options: Optional[Union[BaseDynamicOptions, List[OptionValue]]] = Field(
         None, description="The options of the parameter"
     )
     value: Optional[Any] = Field(
@@ -379,7 +368,7 @@ class Parameter(TypeMetadata, Serializable):
         default: Optional[Union[DefaultParameterType, _MISSING_TYPE]] = _MISSING_VALUE,
         placeholder: Optional[DefaultParameterType] = None,
         description: Optional[str] = None,
-        options: Optional[List[OptionValue]] = None,
+        options: Optional[Union[BaseDynamicOptions, List[OptionValue]]] = None,
         resource_type: ResourceType = ResourceType.INSTANCE,
     ):
         """Build the parameter from the type."""
@@ -435,7 +424,15 @@ class Parameter(TypeMetadata, Serializable):
 
     def to_dict(self) -> Dict:
         """Convert current metadata to json dict."""
-        return self.dict()
+        dict_value = self.dict(exclude={"options"})
+        if not self.options:
+            dict_value["options"] = None
+        elif isinstance(self.options, BaseDynamicOptions):
+            values = self.options.option_values()
+            dict_value["options"] = [value.to_dict() for value in values]
+        else:
+            dict_value["options"] = [value.to_dict() for value in self.options]
+        return dict_value
 
     def to_runnable_parameter(
         self,
@@ -685,6 +682,14 @@ class BaseMetadata(BaseResource):
         split_ids = self.id.split("_")
         return "_".join(split_ids[:-1])
 
+    def to_dict(self) -> Dict:
+        """Convert current metadata to json dict."""
+        dict_value = self.dict(exclude={"parameters"})
+        dict_value["parameters"] = [
+            parameter.to_dict() for parameter in self.parameters
+        ]
+        return dict_value
+
 
 class ResourceMetadata(BaseMetadata, TypeMetadata):
     """The metadata of the resource."""
@@ -939,9 +944,9 @@ class FlowRegistry:
         """Get the registry item by the key."""
         return self._registry.get(key)
 
-    def metadata_list(self) -> List[Union[ViewMetadata, ResourceMetadata]]:
+    def metadata_list(self):
         """Get the metadata list."""
-        return [item.metadata for item in self._registry.values()]
+        return [item.metadata.to_dict() for item in self._registry.values()]
 
 
 _OPERATOR_REGISTRY: FlowRegistry = FlowRegistry()

dbgpt/core/awel/util/parameter_util.py

@@ -0,0 +1,70 @@
+"""The parameter utility."""
+
+import inspect
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List
+
+from dbgpt._private.pydantic import BaseModel, Field, root_validator
+from dbgpt.core.interface.serialization import Serializable
+
+_DEFAULT_DYNAMIC_REGISTRY = {}
+
+
+class OptionValue(Serializable, BaseModel):
+    """The option value of the parameter."""
+
+    label: str = Field(..., description="The label of the option")
+    name: str = Field(..., description="The name of the option")
+    value: Any = Field(..., description="The value of the option")
+
+    def to_dict(self) -> Dict:
+        """Convert current metadata to json dict."""
+        return self.dict()
+
+
+class BaseDynamicOptions(Serializable, BaseModel, ABC):
+    """The base dynamic options."""
+
+    @abstractmethod
+    def option_values(self) -> List[OptionValue]:
+        """Return the option values of the parameter."""
+
+
+class FunctionDynamicOptions(BaseDynamicOptions):
+    """The function dynamic options."""
+
+    func: Callable[[], List[OptionValue]] = Field(
+        ..., description="The function to generate the dynamic options"
+    )
+    func_id: str = Field(
+        ..., description="The unique id of the function to generate the dynamic options"
+    )
+
+    def option_values(self) -> List[OptionValue]:
+        """Return the option values of the parameter."""
+        return self.func()
+
+    @root_validator(pre=True)
+    def pre_fill(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Pre fill the function id."""
+        func = values.get("func")
+        if func is None:
+            raise ValueError(
+                "The function to generate the dynamic options is required."
+            )
+        func_id = _generate_unique_id(func)
+        values["func_id"] = func_id
+        _DEFAULT_DYNAMIC_REGISTRY[func_id] = func
+        return values
+
+    def to_dict(self) -> Dict:
+        """Convert current metadata to json dict."""
+        return {"func_id": self.func_id}
+
+
+def _generate_unique_id(func: Callable) -> str:
+    if func.__name__ == "<lambda>":
+        func_id = f"lambda_{inspect.getfile(func)}_{inspect.getsourcelines(func)}"
+    else:
+        func_id = f"{func.__module__}.{func.__name__}"
+    return func_id

dbgpt/serve/agent/team/layout/agent_operator_resource.py

@@ -4,21 +4,16 @@ from pydantic import BaseModel, Field
 
 from dbgpt._private.pydantic import root_validator
 from dbgpt.agent.agents.agents_manage import agent_manage
-from dbgpt.agent.agents.base_agent_new import ConversableAgent
 from dbgpt.agent.agents.llm.llm import LLMConfig, LLMStrategyType
 from dbgpt.agent.resource.resource_api import AgentResource, ResourceType
 from dbgpt.core import LLMClient
 from dbgpt.core.awel.flow import (
-    IOField,
+    FunctionDynamicOptions,
-    OperatorCategory,
-    OperatorType,
     OptionValue,
     Parameter,
     ResourceCategory,
-    ViewMetadata,
     register_resource,
 )
-from dbgpt.core.interface.operators.prompt_operator import CommonChatPromptTemplate
 
 
 @register_resource(
@@ -115,6 +110,13 @@ class AwelAgentConfig(LLMConfig):
         return values
 
 
+def _agent_resource_option_values() -> List[OptionValue]:
+    return [
+        OptionValue(label=item["name"], name=item["name"], value=item["name"])
+        for item in agent_manage.list_agents()
+    ]
+
+
 @register_resource(
     label="Awel Layout Agent",
     name="agent_operator_agent",
@@ -126,10 +128,7 @@ class AwelAgentConfig(LLMConfig):
             name="agent_profile",
             type=str,
             description="Which agent want use.",
-            options=[
+            options=FunctionDynamicOptions(func=_agent_resource_option_values),
-                OptionValue(label=item["name"], name=item["name"], value=item["name"])
-                for item in agent_manage.list_agents()
-            ],
         ),
         Parameter.build_from(
             label="Role Name",

dbgpt/serve/rag/operators/knowledge_space.py

@@ -13,6 +13,7 @@ from dbgpt.core import (
 )
 from dbgpt.core.awel import JoinOperator, MapOperator
 from dbgpt.core.awel.flow import (
+    FunctionDynamicOptions,
     IOField,
     OperatorCategory,
     OperatorType,
@@ -29,6 +30,15 @@ from dbgpt.storage.vector_store.connector import VectorStoreConnector
 from dbgpt.util.function_utils import rearrange_args_by_type
 
 
+def _load_space_name() -> List[OptionValue]:
+    return [
+        OptionValue(label=space.name, name=space.name, value=space.name)
+        for space in knowledge_space_service.get_knowledge_space(
+            KnowledgeSpaceRequest()
+        )
+    ]
+
+
 class SpaceRetrieverOperator(MapOperator[IN, OUT]):
     """knowledge space retriever operator."""
 
@@ -51,12 +61,7 @@ class SpaceRetrieverOperator(MapOperator[IN, OUT]):
                 "Space Name",
                 "space_name",
                 str,
-                options=[
+                options=FunctionDynamicOptions(func=_load_space_name),
-                    OptionValue(label=space.name, name=space.name, value=space.name)
-                    for space in knowledge_space_service.get_knowledge_space(
-                        KnowledgeSpaceRequest()
-                    )
-                ],
                 optional=False,
                 default=None,
                 description="space name.",