feat:knowledge management

1.merge main branch
2.rebuild knowledge management
3.static js

.dockerignore

@@ -1 +1,2 @@
 models/
+plugins/

.gitignore

@@ -27,7 +27,9 @@ sdist/
 
 var/
 wheels/
-models/*
+models/
+plugins/
+
 pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/

Dockerfile

@@ -1,23 +1,23 @@
 FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
 
+WORKDIR /app
+
 RUN apt-get update && apt-get install -y \
 	git \
 	python3 \
 	pip
 
-	
-WORKDIR /app
-
-COPY . /app
-
-
 # upgrade pip
 RUN pip3 install --upgrade pip
 
+COPY ./requirements.txt /app/requirements.txt
+
 RUN pip install --no-cache-dir -r requirements.txt
 
 RUN python3 -m spacy download zh_core_web_sm
 
-EXPOSE 7860
 
-CMD ["python3", "pilot/server/webserver.py"]
+COPY . /app
+
+EXPOSE 7860
+EXPOSE 8000

Dockerfile-llmserver

@@ -1,21 +0,0 @@
-FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
-
-RUN apt-get update && apt-get install -y \
-	git \
-	python3 \
-	pip
-
-	
-WORKDIR /app
-
-COPY . /app
-
-
-# upgrade pip
-RUN pip3 install --upgrade pip
-
-RUN pip install --no-cache-dir -r requirements.txt
-
-EXPOSE 8000
-
-CMD ["python3", "pilot/server/llmserver.py"]

README.md

@@ -10,7 +10,7 @@
     </a>
   </p>
 
-[**简体中文**](README.zh.md) |[**Discord**](https://discord.gg/hmGPnMSb) |[**Documents**](https://db-gpt.readthedocs.io/en/latest/)|[**Wechat**](https://github.com/csunny/DB-GPT/blob/main/README.zh.md#%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC) 
+[**简体中文**](README.zh.md) |[**Discord**](https://discord.gg/4BNdxm5d) |[**Documents**](https://db-gpt.readthedocs.io/en/latest/)|[**Wechat**](https://github.com/csunny/DB-GPT/blob/main/README.zh.md#%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC) 
 
 </div>
 
@@ -21,10 +21,11 @@ As large models are released and iterated upon, they are becoming increasingly i
 DB-GPT is an experimental open-source project that uses localized GPT large models to interact with your data and environment. With this solution, you can be assured that there is no risk of data leakage, and your data is 100% private and secure.
 
 ## News
+- [2023/06/25]🔥support chatglm2-6b model. [documents](https://db-gpt.readthedocs.io/en/latest/modules/llms.html) 
 - [2023/06/14] support gpt4all model, which can run at M1/M2, or cpu machine. [documents](https://db-gpt.readthedocs.io/en/latest/modules/llms.html) 
 - [2023/06/01]🔥 On the basis of the Vicuna-13B basic model, task chain calls are implemented through plugins. For example, the implementation of creating a database with a single sentence.[demo](./assets/auto_plugin.gif)
 - [2023/06/01]🔥 QLoRA guanaco(7b, 13b, 33b) support.
-- [2023/05/28]🔥 Learning from crawling data from the Internet [demo](./assets/chaturl_en.gif)
+- [2023/05/28] Learning from crawling data from the Internet [demo](./assets/chaturl_en.gif)
 - [2023/05/21] Generate SQL and execute it automatically. [demo](./assets/auto_sql_en.gif)
 - [2023/05/15] Chat with documents. [demo](./assets/new_knownledge_en.gif)
 - [2023/05/06] SQL generation and diagnosis. [demo](./assets/demo_en.gif)
@@ -149,7 +150,7 @@ This project is standing on the shoulders of giants and is not going to work wit
 The MIT License (MIT)
 
 ## Contact Information
-We are working on building a community, if you have any ideas about building the community, feel free to contact us. [Discord](https://discord.gg/hmGPnMSb)
+We are working on building a community, if you have any ideas about building the community, feel free to contact us. [Discord](https://discord.gg/4BNdxm5d)
 
 [![Star History Chart](https://api.star-history.com/svg?repos=csunny/DB-GPT)](https://star-history.com/#csunny/DB-GPT)
 

README.zh.md

@@ -23,6 +23,7 @@ DB-GPT 是一个开源的以数据库为基础的GPT实验项目，使用本地
 
 
 ## 最新发布
+- [2023/06/25]🔥 支持ChatGLM2-6B模型。 [使用文档](https://db-gpt.readthedocs.io/projects/db-gpt-docs-zh-cn/zh_CN/latest/modules/llms.html)
 - [2023/06/14]🔥 支持gpt4all模型，可以在M1/M2 或者CPU机器上运行。 [使用文档](https://db-gpt.readthedocs.io/projects/db-gpt-docs-zh-cn/zh_CN/latest/modules/llms.html)
 - [2023/06/01]🔥 在Vicuna-13B基础模型的基础上，通过插件实现任务链调用。例如单句创建数据库的实现.
 - [2023/06/01]🔥 QLoRA guanaco(原驼)支持, 支持4090运行33B

docker-compose.yml

@@ -16,11 +16,16 @@ services:
   webserver:
     build:
       context: .
-      dockerfile: Dockerfile-webserver
+      dockerfile: Dockerfile
+    command: python3 pilot/server/webserver.py
     environment:
       - MODEL_SERVER=http://llmserver:8000
       - LOCAL_DB_HOST=db
       - WEB_SERVER_PORT=7860
+      - ALLOWLISTED_PLUGINS=db_dashboard
+    depends_on:
+      - db
+      - llmserver
     volumes:
       - ./models:/app/models
       - ./plugins:/app/plugins
@@ -28,16 +33,19 @@ services:
     env_file:
       - .env.template
     ports:
-      - 7860:7860
+      - 7860:7860/tcp
     expose:
-      - 7860
+      - 7860/tcp
     restart: unless-stopped
   llmserver:
     build:
       context: .
-      dockerfile: Dockerfile-llmserver
+      dockerfile: Dockerfile
+    command: python3 pilot/server/llmserver.py
     environment:
       - LOCAL_DB_HOST=db
+    depends_on:
+      - db
     volumes:
       - ./models:/app/models
     env_file:
@@ -50,9 +58,17 @@ services:
         reservations:
           devices:
             - driver: nvidia
-              device_ids: ['1']
+              device_ids: ['0']
               capabilities: [gpu]
-
+  tunnel:
+    image: cloudflare/cloudflared:latest
+    container_name: cloudflared-tunnel
+    restart: unless-stopped
+    environment:
+      - TUNNEL_URL=http://webserver:7860
+    command: tunnel --no-autoupdate
+    depends_on:
+      - webserver
 
 
 volumes:

pilot/configs/model_config.py

@@ -39,6 +39,8 @@ LLM_MODEL_CONFIG = {
     "codet5p-2b": os.path.join(MODEL_PATH, "codet5p-2b"),
     "chatglm-6b-int4": os.path.join(MODEL_PATH, "chatglm-6b-int4"),
     "chatglm-6b": os.path.join(MODEL_PATH, "chatglm-6b"),
+    "chatglm2-6b": os.path.join(MODEL_PATH, "chatglm2-6b"),
+    "chatglm2-6b-int4": os.path.join(MODEL_PATH, "chatglm2-6b-int4"),
     "text2vec-base": os.path.join(MODEL_PATH, "text2vec-base-chinese"),
     "guanaco-33b-merged": os.path.join(MODEL_PATH, "guanaco-33b-merged"),
     "falcon-40b": os.path.join(MODEL_PATH, "falcon-40b"),

pilot/model/llm_out/chatglm_llm.py

@@ -54,7 +54,12 @@ def chatglm_generate_stream(
     try:
         query = messages[-2].split("human:")[1]
     except IndexError:
-        query = messages[-3].split("human:")[1]
+        # fix doc qa: https://github.com/csunny/DB-GPT/issues/274
+        doc_qa_message = messages[-2]
+        if "system:" in doc_qa_message:
+            query = doc_qa_message.split("system:")[1]
+        else:
+            query = messages[-3].split("human:")[1]
     print("Query Message: ", query)
     # output = ""
     # i = 0

pilot/openapi/api_v1/api_v1.py

@@ -17,7 +17,6 @@ from fastapi.responses import JSONResponse, HTMLResponse
 from fastapi.responses import StreamingResponse, FileResponse
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import RequestValidationError
-from sse_starlette.sse import EventSourceResponse
 from typing import List
 
 from pilot.openapi.api_v1.api_view_model import (

pilot/out_parser/base.py

@@ -119,7 +119,7 @@ class BaseOutputParser(ABC):
         else:
             raise ValueError("Model server error!code=" + respObj_ex["error_code"])
 
-    def __extract_json(slef, s):
+    def __extract_json(self, s):
         i = s.index("{")
         count = 1  # 当前所在嵌套深度，即还没闭合的'{'个数
         for j, c in enumerate(s[i + 1 :], start=i + 1):

pilot/source_embedding/knowledge_embedding.py

@@ -35,6 +35,7 @@ class KnowledgeEmbedding:
         self.knowledge_embedding_client.index_to_store(docs)
 
     def read(self):
+        self.knowledge_embedding_client = self.init_knowledge_embedding()
         return self.knowledge_embedding_client.read_batch()
 
     def init_knowledge_embedding(self):

pilot/source_embedding/source_embedding.py

@@ -33,9 +33,6 @@ class SourceEmbedding(ABC):
         self.vector_store_config = vector_store_config
         self.embedding_args = embedding_args
         self.embeddings = vector_store_config["embeddings"]
-        self.vector_client = VectorStoreConnector(
-            CFG.VECTOR_STORE_TYPE, vector_store_config
-        )
 
     @abstractmethod
     @register
@@ -59,11 +56,17 @@ class SourceEmbedding(ABC):
     @register
     def index_to_store(self, docs):
         """index to vector store"""
+        self.vector_client = VectorStoreConnector(
+            CFG.VECTOR_STORE_TYPE, self.vector_store_config
+        )
         self.vector_client.load_document(docs)
 
     @register
     def similar_search(self, doc, topk):
         """vector store similarity_search"""
+        self.vector_client = VectorStoreConnector(
+            CFG.VECTOR_STORE_TYPE, self.vector_store_config
+        )
         try:
             ans = self.vector_client.similar_search(doc, topk)
         except NotEnoughElementsException:
@@ -71,6 +74,9 @@ class SourceEmbedding(ABC):
         return ans
 
     def vector_name_exist(self):
+        self.vector_client = VectorStoreConnector(
+            CFG.VECTOR_STORE_TYPE, self.vector_store_config
+        )
         return self.vector_client.vector_name_exists()
 
     def source_embedding(self):

setup.py

@@ -19,7 +19,7 @@ def parse_requirements(file_name: str) -> List[str]:
 setuptools.setup(
     name="DB-GPT",
     packages=find_packages(),
-    version="0.2.2",
+    version="0.2.3",
     author="csunny",
     author_email="cfqcsunny@gmail.com",
     description="DB-GPT is an experimental open-source project that uses localized GPT large models to interact with your data and environment."

tools/knowledge_init.py

@@ -30,23 +30,26 @@ class LocalKnowledgeInit:
 
     def knowledge_persist(self, file_path):
         """knowledge persist"""
+        docs = []
+        embedding_engine = None
         for root, _, files in os.walk(file_path, topdown=False):
             for file in files:
                 filename = os.path.join(root, file)
-                # docs = self._load_file(filename)
                 ke = KnowledgeEmbedding(
                     knowledge_source=filename,
                     knowledge_type=KnowledgeType.DOCUMENT.value,
                     model_name=self.model_name,
                     vector_store_config=self.vector_store_config,
                 )
-                client = ke.init_knowledge_embedding()
-                client.source_embedding()
+                embedding_engine = ke.init_knowledge_embedding()
+                doc = ke.read()
+                docs.extend(doc)
+        embedding_engine.index_to_store(docs)
         print(f"""begin create {self.vector_store_config["vector_store_name"]} space""")
         space = KnowledgeSpaceRequest
         space.name = self.vector_store_config["vector_store_name"]
-        space.desc = ""
-        space.owner = "knowledge_init.py"
+        space.desc = "knowledge_init.py"
+        space.owner = "DB-GPT"
         knowledge_space_service.create_knowledge_space(space)