Async auto load native plugins; (#218)

1.Async auto load native plugins;
2.Add Config control if auto load plugins, default open, if wan't close,
change in .env

docs/modules/prompts.md

@@ -1,3 +1,5 @@
 # Prompts
 
-Prompt is a very important part of the interaction between the large model and the user, and to a certain extent, it determines the quality and accuracy of the answer generated by the large model. In this project, we will automatically optimize the corresponding prompt according to user input and usage scenarios, making it easier and more efficient for users to use large language models.
+Prompt is a very important part of the interaction between the large model and the user, and to a certain extent, it determines the quality and accuracy of the answer generated by the large model. In this project, we will automatically optimize the corresponding prompt according to user input and usage scenarios, making it easier and more efficient for users to use large language models.
+
+### 1.DB-GPT Prompt 

pilot/common/plugins.py

@@ -5,6 +5,7 @@ import os
 import glob
 import zipfile
 import requests
+import threading
 import datetime
 from pathlib import Path
 from typing import List
@@ -74,32 +75,40 @@ def create_directory_if_not_exists(directory_path: str) -> bool:
 
 
 def load_native_plugins(cfg: Config):
-    print("load_native_plugins")
+    if not cfg.plugins_auto_load:
-    ### TODO 默认拉主分支，后续拉发布版本
+        print("not auto load_native_plugins")
-    branch_name = cfg.plugins_git_branch
+        return
-    native_plugin_repo = "DB-GPT-Plugins"
+    def load_from_git(cfg: Config):
-    url = "https://github.com/csunny/{repo}/archive/{branch}.zip"
+        print("async load_native_plugins")
-    response = requests.get(
+        branch_name = cfg.plugins_git_branch
-        url.format(repo=native_plugin_repo, branch=branch_name),
+        native_plugin_repo = "DB-GPT-Plugins"
-        headers={"Authorization": "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"},
+        url = "https://github.com/csunny/{repo}/archive/{branch}.zip"
-    )
+        try:
+            session = requests.Session()
+            response = session.get(url.format(repo=native_plugin_repo, branch=branch_name),
+                                    headers={'Authorization': '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'})
+
+            if response.status_code == 200:
+                plugins_path_path = Path(PLUGINS_DIR)
+                files = glob.glob(os.path.join(plugins_path_path, f'{native_plugin_repo}*'))
+                for file in files:
+                    os.remove(file)
+                now = datetime.datetime.now()
+                time_str = now.strftime('%Y%m%d%H%M%S')
+                file_name = f"{plugins_path_path}/{native_plugin_repo}-{branch_name}-{time_str}.zip"
+                print(file_name)
+                with open(file_name, "wb") as f:
+                    f.write(response.content)
+                print("save file")
+                cfg.set_plugins(scan_plugins(cfg, cfg.debug_mode))
+            else:
+                print("get file faild，response code：", response.status_code)
+        except Exception as e:
+            print("load plugin from git exception!" + str(e))
+
+    t = threading.Thread(target=load_from_git, args=(cfg,))
+    t.start()
 
-    if response.status_code == 200:
-        plugins_path_path = Path(PLUGINS_DIR)
-        files = glob.glob(os.path.join(plugins_path_path, f"{native_plugin_repo}*"))
-        for file in files:
-            os.remove(file)
-        now = datetime.datetime.now()
-        time_str = now.strftime("%Y%m%d%H%M%S")
-        file_name = (
-            f"{plugins_path_path}/{native_plugin_repo}-{branch_name}-{time_str}.zip"
-        )
-        print(file_name)
-        with open(file_name, "wb") as f:
-            f.write(response.content)
-        print("文件已保存到本地")
-    else:
-        print("获取Release信息失败，状态码为：", response.status_code)
 
 
 def scan_plugins(cfg: Config, debug: bool = False) -> List[AutoGPTPluginTemplate]:

pilot/configs/config.py

@@ -90,6 +90,7 @@ class Config(metaclass=Singleton):
         ### The associated configuration parameters of the plug-in control the loading and use of the plug-in
         self.plugins: List[AutoGPTPluginTemplate] = []
         self.plugins_openai = []
+        self.plugins_auto_load =  os.getenv("AUTO_LOAD_PLUGIN", "True") == "True"
 
         self.plugins_git_branch = os.getenv("PLUGINS_GIT_BRANCH", "plugin_dashboard")
 

pilot/out_parser/base.py

@@ -155,7 +155,7 @@ class BaseOutputParser(ABC):
             cleaned_output = cleaned_output[: -len("```")]
         cleaned_output = cleaned_output.strip()
         if not cleaned_output.startswith("{") or not cleaned_output.endswith("}"):
-            logger.info("illegal json processing")
+            logger.info("illegal json processing:\n" + cleaned_output)
             cleaned_output = self.__extract_json(cleaned_output)
         cleaned_output = (
             cleaned_output.strip()