feat:add rag awel operator view metadata. (#1174)

dbgpt/rag/operators/knowledge.py

@@ -1,26 +1,92 @@
 from typing import Any, List, Optional
 
 from dbgpt.core.awel import MapOperator
+from dbgpt.core.awel.flow import (
+    IOField,
+    OperatorCategory,
+    OptionValue,
+    Parameter,
+    ViewMetadata,
+)
 from dbgpt.core.awel.task.base import IN
 from dbgpt.rag.knowledge.base import Knowledge, KnowledgeType
 from dbgpt.rag.knowledge.factory import KnowledgeFactory
 
 
 class KnowledgeOperator(MapOperator[Any, Any]):
-    """Knowledge Operator."""
+    """Knowledge Factory Operator."""
+
+    metadata = ViewMetadata(
+        label="Knowledge Factory Operator",
+        name="knowledge_operator",
+        category=OperatorCategory.RAG,
+        description="The knowledge operator.",
+        inputs=[
+            IOField.build_from(
+                "knowledge datasource",
+                "knowledge datasource",
+                dict,
+                "knowledge datasource",
+            )
+        ],
+        outputs=[
+            IOField.build_from(
+                "Knowledge",
+                "Knowledge",
+                Knowledge,
+                description="Knowledge",
+            )
+        ],
+        parameters=[
+            Parameter.build_from(
+                label="datasource",
+                name="datasource",
+                type=str,
+                optional=True,
+                default="DOCUMENT",
+                description="datasource",
+            ),
+            Parameter.build_from(
+                label="knowledge_type",
+                name="knowledge type",
+                type=str,
+                optional=True,
+                options=[
+                    OptionValue(
+                        label="DOCUMENT",
+                        name="DOCUMENT",
+                        value=KnowledgeType.DOCUMENT.name,
+                    ),
+                    OptionValue(label="URL", name="URL", value=KnowledgeType.URL.name),
+                    OptionValue(
+                        label="TEXT", name="TEXT", value=KnowledgeType.TEXT.name
+                    ),
+                ],
+                default=KnowledgeType.DOCUMENT.name,
+                description="knowledge type",
+            ),
+        ],
+        documentation_url="https://github.com/openai/openai-python",
+    )
 
     def __init__(
-        self, knowledge_type: Optional[KnowledgeType] = KnowledgeType.DOCUMENT, **kwargs
+        self,
+        datasource: Optional[str] = None,
+        knowledge_type: Optional[str] = KnowledgeType.DOCUMENT.name,
+        **kwargs
     ):
         """Init the query rewrite operator.
         Args:
             knowledge_type: (Optional[KnowledgeType]) The knowledge type.
         """
         super().__init__(**kwargs)
-        self._knowledge_type = knowledge_type
+        self._datasource = datasource
+        self._knowledge_type = KnowledgeType.get_by_value(knowledge_type)
 
     async def map(self, datasource: IN) -> Knowledge:
         """knowledge operator."""
+        if self._datasource:
+            datasource = self._datasource
         return await self.blocking_func_to_async(
             KnowledgeFactory.create, datasource, self._knowledge_type
         )

dbgpt/rag/operators/rewrite.py

@@ -2,6 +2,7 @@ from typing import Any, List, Optional
 
 from dbgpt.core import LLMClient
 from dbgpt.core.awel import MapOperator
+from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata
 from dbgpt.core.awel.task.base import IN
 from dbgpt.rag.retriever.rewrite import QueryRewrite
 
@@ -9,6 +10,59 @@ from dbgpt.rag.retriever.rewrite import QueryRewrite
 class QueryRewriteOperator(MapOperator[Any, Any]):
     """The Rewrite Operator."""
 
+    metadata = ViewMetadata(
+        label="Query Rewrite Operator",
+        name="query_rewrite_operator",
+        category=OperatorCategory.RAG,
+        description="query rewrite operator.",
+        inputs=[
+            IOField.build_from("query_context", "query_context", dict, "query context")
+        ],
+        outputs=[
+            IOField.build_from(
+                "rewritten queries",
+                "queries",
+                List[str],
+                description="rewritten queries",
+            )
+        ],
+        parameters=[
+            Parameter.build_from(
+                "LLM Client",
+                "llm_client",
+                LLMClient,
+                optional=True,
+                default=None,
+                description="The LLM Client.",
+            ),
+            Parameter.build_from(
+                label="model name",
+                name="model_name",
+                type=str,
+                optional=True,
+                default="gpt-3.5-turbo",
+                description="llm model name",
+            ),
+            Parameter.build_from(
+                label="prompt language",
+                name="language",
+                type=str,
+                optional=True,
+                default="en",
+                description="prompt language",
+            ),
+            Parameter.build_from(
+                label="nums",
+                name="nums",
+                type=int,
+                optional=True,
+                default=5,
+                description="rewrite query nums",
+            ),
+        ],
+        documentation_url="https://github.com/openai/openai-python",
+    )
+
     def __init__(
         self,
         llm_client: Optional[LLMClient],

dbgpt/rag/operators/summary.py

@@ -1,12 +1,77 @@
 from typing import Any, Optional
 
 from dbgpt.core import LLMClient
+from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata
 from dbgpt.core.awel.task.base import IN
+from dbgpt.rag.knowledge.base import Knowledge
 from dbgpt.serve.rag.assembler.summary import SummaryAssembler
 from dbgpt.serve.rag.operators.base import AssemblerOperator
 
 
 class SummaryAssemblerOperator(AssemblerOperator[Any, Any]):
+    metadata = ViewMetadata(
+        label="Summary Operator",
+        name="summary_assembler_operator",
+        category=OperatorCategory.RAG,
+        description="The summary assembler operator.",
+        inputs=[
+            IOField.build_from(
+                "Knowledge", "knowledge", Knowledge, "knowledge datasource"
+            )
+        ],
+        outputs=[
+            IOField.build_from(
+                "document summary",
+                "summary",
+                str,
+                description="document summary",
+            )
+        ],
+        parameters=[
+            Parameter.build_from(
+                "LLM Client",
+                "llm_client",
+                LLMClient,
+                optional=True,
+                default=None,
+                description="The LLM Client.",
+            ),
+            Parameter.build_from(
+                label="model name",
+                name="model_name",
+                type=str,
+                optional=True,
+                default="gpt-3.5-turbo",
+                description="llm model name",
+            ),
+            Parameter.build_from(
+                label="prompt language",
+                name="language",
+                type=str,
+                optional=True,
+                default="en",
+                description="prompt language",
+            ),
+            Parameter.build_from(
+                label="max_iteration_with_llm",
+                name="max_iteration_with_llm",
+                type=int,
+                optional=True,
+                default=5,
+                description="prompt language",
+            ),
+            Parameter.build_from(
+                label="concurrency_limit_with_llm",
+                name="concurrency_limit_with_llm",
+                type=int,
+                optional=True,
+                default=3,
+                description="The concurrency limit with llm",
+            ),
+        ],
+        documentation_url="https://github.com/openai/openai-python",
+    )
+
     def __init__(
         self,
         llm_client: Optional[LLMClient],