From c93fcaaed8157e9b4f5e1cf4f471f58a79d0e14c Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 16:38:45 +0800
Subject: [PATCH 1/8] fix:restore webserver db choose

---
 pilot/server/webserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pilot/server/webserver.py b/pilot/server/webserver.py
index b6c1e2cc3..d39637d89 100644
--- a/pilot/server/webserver.py
+++ b/pilot/server/webserver.py
@@ -703,7 +703,7 @@ if __name__ == "__main__":
     # init server config
     args = parser.parse_args()
     server_init(args)
-
+    dbs = CFG.local_db.get_database_list()
     if args.new:
         import uvicorn
 

From 770c4ae420b8d3d9076246c06c0bea74e9bbe44e Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 20:06:38 +0800
Subject: [PATCH 2/8] fix:embedding chunk update integrate
 RecursiveCharacterTextSplitter

---
 pilot/embedding_engine/markdown_embedding.py | 15 +++++++++------
 pilot/embedding_engine/pdf_embedding.py      | 15 +++++++++------
 pilot/embedding_engine/ppt_embedding.py      | 15 +++++++++------
 pilot/embedding_engine/url_embedding.py      | 15 +++++++++------
 pilot/embedding_engine/word_embedding.py     | 15 +++++++++------
 5 files changed, 45 insertions(+), 30 deletions(-)

diff --git a/pilot/embedding_engine/markdown_embedding.py b/pilot/embedding_engine/markdown_embedding.py
index 0d70ba34f..225cdad82 100644
--- a/pilot/embedding_engine/markdown_embedding.py
+++ b/pilot/embedding_engine/markdown_embedding.py
@@ -6,7 +6,7 @@ from typing import List
 import markdown
 from bs4 import BeautifulSoup
 from langchain.schema import Document
-from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter
+from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.configs.config import Config
 from pilot.embedding_engine import SourceEmbedding, register
@@ -37,11 +37,14 @@ class MarkdownEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = SpacyTextSplitter(
-                pipeline="zh_core_web_sm",
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
-                chunk_overlap=100,
-            )
+            try:
+                text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50)
         return loader.load_and_split(text_splitter)
 
     @register
diff --git a/pilot/embedding_engine/pdf_embedding.py b/pilot/embedding_engine/pdf_embedding.py
index 2b8f244e3..3a47e115e 100644
--- a/pilot/embedding_engine/pdf_embedding.py
+++ b/pilot/embedding_engine/pdf_embedding.py
@@ -4,7 +4,7 @@ from typing import List
 
 from langchain.document_loaders import PyPDFLoader
 from langchain.schema import Document
-from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter
+from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.configs.config import Config
 from pilot.embedding_engine import SourceEmbedding, register
@@ -40,11 +40,14 @@ class PDFEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = SpacyTextSplitter(
-                pipeline="zh_core_web_sm",
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
-                chunk_overlap=100,
-            )
+            try:
+                text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50)
         return loader.load_and_split(text_splitter)
 
     @register
diff --git a/pilot/embedding_engine/ppt_embedding.py b/pilot/embedding_engine/ppt_embedding.py
index da4390849..2ddc583f9 100644
--- a/pilot/embedding_engine/ppt_embedding.py
+++ b/pilot/embedding_engine/ppt_embedding.py
@@ -4,7 +4,7 @@ from typing import List
 
 from langchain.document_loaders import UnstructuredPowerPointLoader
 from langchain.schema import Document
-from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter
+from langchain.text_splitter import SpacyTextSplitter, CharacterTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.configs.config import Config
 from pilot.embedding_engine import SourceEmbedding, register
@@ -38,11 +38,14 @@ class PPTEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = SpacyTextSplitter(
-                pipeline="zh_core_web_sm",
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
-                chunk_overlap=100,
-            )
+            try:
+                text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50)
         return loader.load_and_split(text_splitter)
 
     @register
diff --git a/pilot/embedding_engine/url_embedding.py b/pilot/embedding_engine/url_embedding.py
index ce9e5360d..273b5e777 100644
--- a/pilot/embedding_engine/url_embedding.py
+++ b/pilot/embedding_engine/url_embedding.py
@@ -3,7 +3,7 @@ from typing import List
 from bs4 import BeautifulSoup
 from langchain.document_loaders import WebBaseLoader
 from langchain.schema import Document
-from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.configs.config import Config
 from pilot.configs.model_config import KNOWLEDGE_CHUNK_SPLIT_SIZE
@@ -33,11 +33,14 @@ class URLEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = SpacyTextSplitter(
-                pipeline="zh_core_web_sm",
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
-                chunk_overlap=100,
-            )
+            try:
+                text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50)
         return loader.load_and_split(text_splitter)
 
     @register
diff --git a/pilot/embedding_engine/word_embedding.py b/pilot/embedding_engine/word_embedding.py
index 55988a240..efb337a9a 100644
--- a/pilot/embedding_engine/word_embedding.py
+++ b/pilot/embedding_engine/word_embedding.py
@@ -4,7 +4,7 @@ from typing import List
 
 from langchain.document_loaders import UnstructuredWordDocumentLoader
 from langchain.schema import Document
-from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.configs.config import Config
 from pilot.embedding_engine import SourceEmbedding, register
@@ -32,11 +32,14 @@ class WordEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = SpacyTextSplitter(
-                pipeline="zh_core_web_sm",
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
-                chunk_overlap=100,
-            )
+            try:
+                text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50)
         return loader.load_and_split(text_splitter)
 
     @register

From be9e826edf1de180ae1531f1906b84a744ad3700 Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 21:05:23 +0800
Subject: [PATCH 3/8] refactor: code refactor

---
 pilot/openapi/api_v1/api_v1.py                         |  4 ++--
 pilot/server/dbgpt_server.py                           |  2 +-
 pilot/{openapi => server}/knowledge/__init__.py        |  0
 .../knowledge/api.py}                                  |  8 ++++----
 .../knowledge/chunk_dao.py}                            |  0
 .../knowledge/document_dao.py}                         |  0
 .../{openapi => server}/knowledge/request/__init__.py  |  0
 .../knowledge/request/request.py}                      |  0
 .../knowledge/request/response.py}                     |  0
 .../knowledge/service.py}                              | 10 +++++-----
 .../knowledge/space_dao.py}                            |  2 +-
 11 files changed, 13 insertions(+), 13 deletions(-)
 rename pilot/{openapi => server}/knowledge/__init__.py (100%)
 rename pilot/{openapi/knowledge/knowledge_controller.py => server/knowledge/api.py} (94%)
 rename pilot/{openapi/knowledge/document_chunk_dao.py => server/knowledge/chunk_dao.py} (100%)
 rename pilot/{openapi/knowledge/knowledge_document_dao.py => server/knowledge/document_dao.py} (100%)
 rename pilot/{openapi => server}/knowledge/request/__init__.py (100%)
 rename pilot/{openapi/knowledge/request/knowledge_request.py => server/knowledge/request/request.py} (100%)
 rename pilot/{openapi/knowledge/request/knowledge_response.py => server/knowledge/request/response.py} (100%)
 rename pilot/{openapi/knowledge/knowledge_service.py => server/knowledge/service.py} (95%)
 rename pilot/{openapi/knowledge/knowledge_space_dao.py => server/knowledge/space_dao.py} (97%)

diff --git a/pilot/openapi/api_v1/api_v1.py b/pilot/openapi/api_v1/api_v1.py
index 7f6f66953..3838c8dbe 100644
--- a/pilot/openapi/api_v1/api_v1.py
+++ b/pilot/openapi/api_v1/api_v1.py
@@ -26,8 +26,8 @@ from pilot.openapi.api_v1.api_view_model import (
     ChatSceneVo,
 )
 from pilot.configs.config import Config
-from pilot.openapi.knowledge.knowledge_service import KnowledgeService
-from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest
+from pilot.server.knowledge.service import KnowledgeService
+from pilot.server.knowledge.request.request import KnowledgeSpaceRequest
 
 from pilot.scene.base_chat import BaseChat
 from pilot.scene.base import ChatScene
diff --git a/pilot/server/dbgpt_server.py b/pilot/server/dbgpt_server.py
index f72f04dd7..4996e7c4a 100644
--- a/pilot/server/dbgpt_server.py
+++ b/pilot/server/dbgpt_server.py
@@ -23,7 +23,7 @@ from fastapi import FastAPI, applications
 from fastapi.openapi.docs import get_swagger_ui_html
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
-from pilot.openapi.knowledge.knowledge_controller import router as knowledge_router
+from pilot.server.knowledge.api import router as knowledge_router
 
 
 from pilot.openapi.api_v1.api_v1 import router as api_v1, validation_exception_handler
diff --git a/pilot/openapi/knowledge/__init__.py b/pilot/server/knowledge/__init__.py
similarity index 100%
rename from pilot/openapi/knowledge/__init__.py
rename to pilot/server/knowledge/__init__.py
diff --git a/pilot/openapi/knowledge/knowledge_controller.py b/pilot/server/knowledge/api.py
similarity index 94%
rename from pilot/openapi/knowledge/knowledge_controller.py
rename to pilot/server/knowledge/api.py
index 26e79609f..887bbadb2 100644
--- a/pilot/openapi/knowledge/knowledge_controller.py
+++ b/pilot/server/knowledge/api.py
@@ -2,7 +2,7 @@ import os
 import shutil
 from tempfile import NamedTemporaryFile
 
-from fastapi import APIRouter, File, UploadFile, Request, Form
+from fastapi import APIRouter, File, UploadFile, Form
 
 from langchain.embeddings import HuggingFaceEmbeddings
 
@@ -12,8 +12,8 @@ from pilot.configs.model_config import LLM_MODEL_CONFIG, KNOWLEDGE_UPLOAD_ROOT_P
 from pilot.openapi.api_v1.api_view_model import Result
 from pilot.embedding_engine.knowledge_embedding import KnowledgeEmbedding
 
-from pilot.openapi.knowledge.knowledge_service import KnowledgeService
-from pilot.openapi.knowledge.request.knowledge_request import (
+from pilot.server.knowledge.service import KnowledgeService
+from pilot.server.knowledge.request.request import (
     KnowledgeQueryRequest,
     KnowledgeQueryResponse,
     KnowledgeDocumentRequest,
@@ -22,7 +22,7 @@ from pilot.openapi.knowledge.request.knowledge_request import (
     DocumentQueryRequest,
 )
 
-from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest
+from pilot.server.knowledge.request.request import KnowledgeSpaceRequest
 
 CFG = Config()
 router = APIRouter()
diff --git a/pilot/openapi/knowledge/document_chunk_dao.py b/pilot/server/knowledge/chunk_dao.py
similarity index 100%
rename from pilot/openapi/knowledge/document_chunk_dao.py
rename to pilot/server/knowledge/chunk_dao.py
diff --git a/pilot/openapi/knowledge/knowledge_document_dao.py b/pilot/server/knowledge/document_dao.py
similarity index 100%
rename from pilot/openapi/knowledge/knowledge_document_dao.py
rename to pilot/server/knowledge/document_dao.py
diff --git a/pilot/openapi/knowledge/request/__init__.py b/pilot/server/knowledge/request/__init__.py
similarity index 100%
rename from pilot/openapi/knowledge/request/__init__.py
rename to pilot/server/knowledge/request/__init__.py
diff --git a/pilot/openapi/knowledge/request/knowledge_request.py b/pilot/server/knowledge/request/request.py
similarity index 100%
rename from pilot/openapi/knowledge/request/knowledge_request.py
rename to pilot/server/knowledge/request/request.py
diff --git a/pilot/openapi/knowledge/request/knowledge_response.py b/pilot/server/knowledge/request/response.py
similarity index 100%
rename from pilot/openapi/knowledge/request/knowledge_response.py
rename to pilot/server/knowledge/request/response.py
diff --git a/pilot/openapi/knowledge/knowledge_service.py b/pilot/server/knowledge/service.py
similarity index 95%
rename from pilot/openapi/knowledge/knowledge_service.py
rename to pilot/server/knowledge/service.py
index b6308a81a..35f644ffb 100644
--- a/pilot/openapi/knowledge/knowledge_service.py
+++ b/pilot/server/knowledge/service.py
@@ -5,19 +5,19 @@ from pilot.configs.config import Config
 from pilot.configs.model_config import LLM_MODEL_CONFIG
 from pilot.embedding_engine.knowledge_embedding import KnowledgeEmbedding
 from pilot.logs import logger
-from pilot.openapi.knowledge.document_chunk_dao import (
+from pilot.server.knowledge.chunk_dao import (
     DocumentChunkEntity,
     DocumentChunkDao,
 )
-from pilot.openapi.knowledge.knowledge_document_dao import (
+from pilot.server.knowledge.document_dao import (
     KnowledgeDocumentDao,
     KnowledgeDocumentEntity,
 )
-from pilot.openapi.knowledge.knowledge_space_dao import (
+from pilot.server.knowledge.space_dao import (
     KnowledgeSpaceDao,
     KnowledgeSpaceEntity,
 )
-from pilot.openapi.knowledge.request.knowledge_request import (
+from pilot.server.knowledge.request.request import (
     KnowledgeSpaceRequest,
     KnowledgeDocumentRequest,
     DocumentQueryRequest,
@@ -25,7 +25,7 @@ from pilot.openapi.knowledge.request.knowledge_request import (
 )
 from enum import Enum
 
-from pilot.openapi.knowledge.request.knowledge_response import (
+from pilot.server.knowledge.request.response import (
     ChunkQueryResponse,
     DocumentQueryResponse,
 )
diff --git a/pilot/openapi/knowledge/knowledge_space_dao.py b/pilot/server/knowledge/space_dao.py
similarity index 97%
rename from pilot/openapi/knowledge/knowledge_space_dao.py
rename to pilot/server/knowledge/space_dao.py
index 16def1e99..d4f6fbba6 100644
--- a/pilot/openapi/knowledge/knowledge_space_dao.py
+++ b/pilot/server/knowledge/space_dao.py
@@ -5,7 +5,7 @@ from sqlalchemy.ext.declarative import declarative_base
 
 from pilot.configs.config import Config
 
-from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest
+from pilot.server.knowledge.request.request import KnowledgeSpaceRequest
 from sqlalchemy.orm import sessionmaker
 
 CFG = Config()

From ad21da61d0c0a13232a14a0dafa4a25b052eb3d4 Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 21:40:30 +0800
Subject: [PATCH 4/8] fix:embedding chunk update integrate

---
 tools/knowledge_init.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/knowledge_init.py b/tools/knowledge_init.py
index 34460e398..30446cc6c 100644
--- a/tools/knowledge_init.py
+++ b/tools/knowledge_init.py
@@ -6,8 +6,8 @@ import sys
 sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 
 from pilot.embedding_engine.knowledge_type import KnowledgeType
-from pilot.openapi.knowledge.knowledge_service import KnowledgeService
-from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest
+from pilot.server.knowledge.service import KnowledgeService
+from pilot.server.knowledge.request.request import KnowledgeSpaceRequest
 
 
 from pilot.configs.config import Config

From 161fe4b358d73b5d325a4a1c6b60c69ad682b01a Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 23:50:34 +0800
Subject: [PATCH 5/8] refactor:knowledge api refactor 1.knowledge python file
 refactor 2.source embedding return vector_ids

---
 assets/schema/knowledge_management.sql                   | 6 +++---
 pilot/embedding_engine/source_embedding.py               | 2 +-
 pilot/server/knowledge/{chunk_dao.py => chunk_db.py}     | 0
 .../server/knowledge/{document_dao.py => document_db.py} | 0
 pilot/server/knowledge/service.py                        | 9 +++++----
 pilot/server/knowledge/{space_dao.py => space_db.py}     | 0
 6 files changed, 9 insertions(+), 8 deletions(-)
 rename pilot/server/knowledge/{chunk_dao.py => chunk_db.py} (100%)
 rename pilot/server/knowledge/{document_dao.py => document_db.py} (100%)
 rename pilot/server/knowledge/{space_dao.py => space_db.py} (100%)

diff --git a/assets/schema/knowledge_management.sql b/assets/schema/knowledge_management.sql
index 2c50e42b4..ceaa566da 100644
--- a/assets/schema/knowledge_management.sql
+++ b/assets/schema/knowledge_management.sql
@@ -8,7 +8,7 @@ CREATE TABLE `knowledge_space` (
   `gmt_modified` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
   PRIMARY KEY (`id`),
   KEY `idx_name` (`name`) COMMENT 'index:idx_name'
-) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='knowledge space table';
+) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge space table';
 
 CREATE TABLE `knowledge_document` (
   `id` int NOT NULL AUTO_INCREMENT COMMENT 'auto increment id',
@@ -25,7 +25,7 @@ CREATE TABLE `knowledge_document` (
   `gmt_modified` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
   PRIMARY KEY (`id`),
   KEY `idx_doc_name` (`doc_name`) COMMENT 'index:idx_doc_name'
-) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='knowledge document table';
+) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge document table';
 
 CREATE TABLE `document_chunk` (
   `id` int NOT NULL AUTO_INCREMENT COMMENT 'auto increment id',
@@ -38,4 +38,4 @@ CREATE TABLE `document_chunk` (
   `gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
   PRIMARY KEY (`id`),
   KEY `idx_document_id` (`document_id`) COMMENT 'index:document_id'
-) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='knowledge document chunk detail'
\ No newline at end of file
+) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge document chunk detail'
\ No newline at end of file
diff --git a/pilot/embedding_engine/source_embedding.py b/pilot/embedding_engine/source_embedding.py
index 372e35c22..6d7500007 100644
--- a/pilot/embedding_engine/source_embedding.py
+++ b/pilot/embedding_engine/source_embedding.py
@@ -59,7 +59,7 @@ class SourceEmbedding(ABC):
         self.vector_client = VectorStoreConnector(
             CFG.VECTOR_STORE_TYPE, self.vector_store_config
         )
-        self.vector_client.load_document(docs)
+        return self.vector_client.load_document(docs)
 
     @register
     def similar_search(self, doc, topk):
diff --git a/pilot/server/knowledge/chunk_dao.py b/pilot/server/knowledge/chunk_db.py
similarity index 100%
rename from pilot/server/knowledge/chunk_dao.py
rename to pilot/server/knowledge/chunk_db.py
diff --git a/pilot/server/knowledge/document_dao.py b/pilot/server/knowledge/document_db.py
similarity index 100%
rename from pilot/server/knowledge/document_dao.py
rename to pilot/server/knowledge/document_db.py
diff --git a/pilot/server/knowledge/service.py b/pilot/server/knowledge/service.py
index 35f644ffb..4d99be4e3 100644
--- a/pilot/server/knowledge/service.py
+++ b/pilot/server/knowledge/service.py
@@ -5,15 +5,15 @@ from pilot.configs.config import Config
 from pilot.configs.model_config import LLM_MODEL_CONFIG
 from pilot.embedding_engine.knowledge_embedding import KnowledgeEmbedding
 from pilot.logs import logger
-from pilot.server.knowledge.chunk_dao import (
+from pilot.server.knowledge.chunk_db import (
     DocumentChunkEntity,
     DocumentChunkDao,
 )
-from pilot.server.knowledge.document_dao import (
+from pilot.server.knowledge.document_db import (
     KnowledgeDocumentDao,
     KnowledgeDocumentEntity,
 )
-from pilot.server.knowledge.space_dao import (
+from pilot.server.knowledge.space_db import (
     KnowledgeSpaceDao,
     KnowledgeSpaceEntity,
 )
@@ -195,7 +195,8 @@ class KnowledgeService:
             vector_ids = client.knowledge_embedding_batch(chunk_docs)
             doc.status = SyncStatus.FINISHED.name
             doc.result = "document embedding success"
-            doc.vector_ids = ",".join(vector_ids)
+            if vector_ids is not None:
+                doc.vector_ids = ",".join(vector_ids)
             logger.info(f"async document embedding, success:{doc.doc_name}")
         except Exception as e:
             doc.status = SyncStatus.FAILED.name
diff --git a/pilot/server/knowledge/space_dao.py b/pilot/server/knowledge/space_db.py
similarity index 100%
rename from pilot/server/knowledge/space_dao.py
rename to pilot/server/knowledge/space_db.py

From 064b08dd2381e8adb48ee39df1d809de7680b74c Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Tue, 4 Jul 2023 11:39:57 +0800
Subject: [PATCH 6/8] feat:space return doc_count

---
 pilot/server/knowledge/request/response.py | 14 ++++++++++++++
 pilot/server/knowledge/service.py          | 15 +++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/pilot/server/knowledge/request/response.py b/pilot/server/knowledge/request/response.py
index 7fbf36155..98dcbcc76 100644
--- a/pilot/server/knowledge/request/response.py
+++ b/pilot/server/knowledge/request/response.py
@@ -21,3 +21,17 @@ class DocumentQueryResponse(BaseModel):
     total: int = None
     """page: current page"""
     page: int = None
+
+
+class SpaceQueryResponse(BaseModel):
+    """data: data"""
+
+    name: str = None
+    """vector_type: vector type"""
+    vector_type: str = None
+    """desc: description"""
+    desc: str = None
+    """owner: owner"""
+    owner: str = None
+    """doc_count: doc_count"""
+    doc_count: int = None
diff --git a/pilot/server/knowledge/service.py b/pilot/server/knowledge/service.py
index 4d99be4e3..f076669c9 100644
--- a/pilot/server/knowledge/service.py
+++ b/pilot/server/knowledge/service.py
@@ -27,7 +27,7 @@ from enum import Enum
 
 from pilot.server.knowledge.request.response import (
     ChunkQueryResponse,
-    DocumentQueryResponse,
+    DocumentQueryResponse, SpaceQueryResponse,
 )
 
 knowledge_space_dao = KnowledgeSpaceDao()
@@ -86,7 +86,18 @@ class KnowledgeService:
         query = KnowledgeSpaceEntity(
             name=request.name, vector_type=request.vector_type, owner=request.owner
         )
-        return knowledge_space_dao.get_knowledge_space(query)
+        res = SpaceQueryResponse()
+        space: KnowledgeSpaceEntity = knowledge_space_dao.get_knowledge_space(query)
+        res.name = space.name
+        res.vector_type = space.vector_type
+        res.desc = space.desc
+        query = KnowledgeDocumentEntity(
+            space=space,
+        )
+        res.doc_count = knowledge_document_dao.get_knowledge_documents_count(query)
+        return res
+
+
 
     """get knowledge get_knowledge_documents"""
 

From 937e5f4528e7f9da82b71a1ea9ca9ab8531550ad Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Tue, 4 Jul 2023 16:40:47 +0800
Subject: [PATCH 7/8] fix:knowledge init try exception

---
 tools/knowledge_init.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tools/knowledge_init.py b/tools/knowledge_init.py
index 30446cc6c..6f46b713f 100644
--- a/tools/knowledge_init.py
+++ b/tools/knowledge_init.py
@@ -46,12 +46,18 @@ class LocalKnowledgeInit:
                 docs.extend(doc)
         embedding_engine.index_to_store(docs)
         print(f"""begin create {self.vector_store_config["vector_store_name"]} space""")
-        space = KnowledgeSpaceRequest
-        space.name = self.vector_store_config["vector_store_name"]
-        space.desc = "knowledge_init.py"
-        space.vector_type = CFG.VECTOR_STORE_TYPE
-        space.owner = "DB-GPT"
-        knowledge_space_service.create_knowledge_space(space)
+        try:
+            space = KnowledgeSpaceRequest
+            space.name = self.vector_store_config["vector_store_name"]
+            space.desc = "knowledge_init.py"
+            space.vector_type = CFG.VECTOR_STORE_TYPE
+            space.owner = "DB-GPT"
+            knowledge_space_service.create_knowledge_space(space)
+        except Exception as e:
+            if "have already named" in str(e):
+                print(f"Warning: you have already named {space.name}")
+            else:
+                raise e
 
 
 if __name__ == "__main__":

From 17e276b76ffeea80cade7cd16512165c139bfa46 Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Tue, 4 Jul 2023 17:59:33 +0800
Subject: [PATCH 8/8] fix:merge web-api

---
 pilot/server/knowledge/service.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/pilot/server/knowledge/service.py b/pilot/server/knowledge/service.py
index f076669c9..ef8cd3d89 100644
--- a/pilot/server/knowledge/service.py
+++ b/pilot/server/knowledge/service.py
@@ -86,18 +86,7 @@ class KnowledgeService:
         query = KnowledgeSpaceEntity(
             name=request.name, vector_type=request.vector_type, owner=request.owner
         )
-        res = SpaceQueryResponse()
-        space: KnowledgeSpaceEntity = knowledge_space_dao.get_knowledge_space(query)
-        res.name = space.name
-        res.vector_type = space.vector_type
-        res.desc = space.desc
-        query = KnowledgeDocumentEntity(
-            space=space,
-        )
-        res.doc_count = knowledge_document_dao.get_knowledge_documents_count(query)
-        return res
-
-
+        return knowledge_space_dao.get_knowledge_space(query)
 
     """get knowledge get_knowledge_documents"""