doc:knowledge docs update

docs/modules/knowledge/markdown/markdown_embedding.md

@@ -6,13 +6,14 @@ inheriting the SourceEmbedding
 
 ```
 class  MarkdownEmbedding(SourceEmbedding):
-    """pdf embedding for read pdf document."""
+    """pdf embedding for read markdown document."""
 
-    def __init__(self, file_path, vector_store_config):
+    def __init__(self, file_path, vector_store_config, text_splitter):
-        """Initialize with pdf path."""
+        """Initialize with markdown path."""
-        super().__init__(file_path, vector_store_config)
+        super().__init__(file_path, vector_store_config, text_splitter)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.text_splitter = text_splitter or Nore
 ```
 implement read() and data_process()
 read() method allows you to read data and split data into chunk
@@ -22,12 +23,19 @@ read() method allows you to read data and split data into chunk
     def read(self):
         """Load from markdown path."""
         loader = EncodeTextLoader(self.file_path)
-        textsplitter = SpacyTextSplitter(
+        if self.text_splitter is None:
+            try:
+                self.text_splitter = SpacyTextSplitter(
                     pipeline="zh_core_web_sm",
-            chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_size=100,
                     chunk_overlap=100,
                 )
-        return loader.load_and_split(textsplitter)
+            except Exception:
+                self.text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return loader.load_and_split(self.text_splitter)
 ```
 
 data_process() method allows you to pre processing your ways

docs/modules/knowledge/pdf/pdf_embedding.md

@@ -7,11 +7,12 @@ inheriting the SourceEmbedding
 class PDFEmbedding(SourceEmbedding):
     """pdf embedding for read pdf document."""
 
-    def __init__(self, file_path, vector_store_config):
+    def __init__(self, file_path, vector_store_config, text_splitter):
         """Initialize with pdf path."""
-        super().__init__(file_path, vector_store_config)
+        super().__init__(file_path, vector_store_config, text_splitter)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.text_splitter = text_splitter or Nore
 ```
 
 implement read() and data_process()
@@ -21,15 +22,19 @@ read() method allows you to read data and split data into chunk
     def read(self):
         """Load from pdf path."""
         loader = PyPDFLoader(self.file_path)
-        # textsplitter = CHNDocumentSplitter(
+        if self.text_splitter is None:
-        #     pdf=True, sentence_size=CFG.KNOWLEDGE_CHUNK_SIZE
+            try:
-        # )
+                self.text_splitter = SpacyTextSplitter(
-        textsplitter = SpacyTextSplitter(
                     pipeline="zh_core_web_sm",
-            chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_size=100,
                     chunk_overlap=100,
                 )
-        return loader.load_and_split(textsplitter)
+            except Exception:
+                self.text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return loader.load_and_split(self.text_splitter)
 ```
 data_process() method allows you to pre processing your ways
 ```

docs/modules/knowledge/ppt/ppt_embedding.md

@@ -7,11 +7,17 @@ inheriting the SourceEmbedding
 class PPTEmbedding(SourceEmbedding):
     """ppt embedding for read ppt document."""
 
-    def __init__(self, file_path, vector_store_config):
+        def __init__(
-        """Initialize with pdf path."""
+        self,
-        super().__init__(file_path, vector_store_config)
+        file_path,
+        vector_store_config,
+        text_splitter: Optional[TextSplitter] = None,
+    ):
+        """Initialize ppt word path."""
+        super().__init__(file_path, vector_store_config, text_splitter=None)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.text_splitter = text_splitter or None
 ```
 
 implement read() and data_process()
@@ -21,12 +27,19 @@ read() method allows you to read data and split data into chunk
     def read(self):
         """Load from ppt path."""
         loader = UnstructuredPowerPointLoader(self.file_path)
-        textsplitter = SpacyTextSplitter(
+        if self.text_splitter is None:
+            try:
+                self.text_splitter = SpacyTextSplitter(
                     pipeline="zh_core_web_sm",
-            chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                    chunk_size=100,
-            chunk_overlap=200,
+                    chunk_overlap=100,
                 )
-        return loader.load_and_split(textsplitter)
+            except Exception:
+                self.text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return loader.load_and_split(self.text_splitter)
 ```
 data_process() method allows you to pre processing your ways
 ```

docs/modules/knowledge/url/url_embedding.md

@@ -7,11 +7,17 @@ inheriting the SourceEmbedding
 class URLEmbedding(SourceEmbedding):
     """url embedding for read url document."""
 
-    def __init__(self, file_path, vector_store_config):
+        def __init__(
-        """Initialize with url path."""
+        self,
-        super().__init__(file_path, vector_store_config)
+        file_path,
+        vector_store_config,
+        text_splitter: Optional[TextSplitter] = None,
+    ):
+        """Initialize url word path."""
+        super().__init__(file_path, vector_store_config, text_splitter=None)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.text_splitter = text_splitter or None
 ```
 
 implement read() and data_process()
@@ -21,15 +27,19 @@ read() method allows you to read data and split data into chunk
     def read(self):
         """Load from url path."""
         loader = WebBaseLoader(web_path=self.file_path)
-        if CFG.LANGUAGE == "en":
+        if self.text_splitter is None:
-            text_splitter = CharacterTextSplitter(
+            try:
-                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                self.text_splitter = SpacyTextSplitter(
-                chunk_overlap=20,
+                    pipeline="zh_core_web_sm",
-                length_function=len,
+                    chunk_size=100,
+                    chunk_overlap=100,
                 )
-        else:
+            except Exception:
-            text_splitter = CHNDocumentSplitter(pdf=True, sentence_size=1000)
+                self.text_splitter = RecursiveCharacterTextSplitter(
-        return loader.load_and_split(text_splitter)
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return loader.load_and_split(self.text_splitter)
 ```
 data_process() method allows you to pre processing your ways
 ```

docs/modules/knowledge/word/word_embedding.md

@@ -7,11 +7,12 @@ inheriting the SourceEmbedding
 class WordEmbedding(SourceEmbedding):
     """word embedding for read word document."""
 
-    def __init__(self, file_path, vector_store_config):
+    def __init__(self, file_path, vector_store_config, text_splitter):
-        """Initialize with word path."""
+        """Initialize with pdf path."""
-        super().__init__(file_path, vector_store_config)
+        super().__init__(file_path, vector_store_config, text_splitter)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.text_splitter = text_splitter or Nore
 ```
 
 implement read() and data_process()
@@ -21,10 +22,19 @@ read() method allows you to read data and split data into chunk
     def read(self):
         """Load from word path."""
         loader = UnstructuredWordDocumentLoader(self.file_path)
-        textsplitter = CHNDocumentSplitter(
+        if self.text_splitter is None:
-            pdf=True, sentence_size=CFG.KNOWLEDGE_CHUNK_SIZE
+            try:
+                self.text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=100,
+                    chunk_overlap=100,
                 )
-        return loader.load_and_split(textsplitter)
+            except Exception:
+                self.text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return loader.load_and_split(self.text_splitter)
 ```
 data_process() method allows you to pre processing your ways
 ```

pilot/embedding_engine/embedding_engine.py

@@ -8,6 +8,13 @@ from pilot.vector_store.connector import VectorStoreConnector
 
 
 class EmbeddingEngine:
+    """EmbeddingEngine provide a chain process include(read->text_split->data_process->index_store) for knowledge document embedding into vector store.
+    1.knowledge_embedding:knowledge document source into vector store.(Chroma, Milvus, Weaviate)
+    2.similar_search: similarity search from vector_store
+    how to use reference:https://db-gpt.readthedocs.io/en/latest/modules/knowledge.html
+    how to integrate:https://db-gpt.readthedocs.io/en/latest/modules/knowledge/pdf/pdf_embedding.html
+    """
+
     def __init__(
         self,
         model_name,
@@ -24,14 +31,17 @@ class EmbeddingEngine:
         self.vector_store_config["embeddings"] = self.embeddings
 
     def knowledge_embedding(self):
+        """source embedding is chain process.read->text_split->data_process->index_store"""
         self.knowledge_embedding_client = self.init_knowledge_embedding()
         self.knowledge_embedding_client.source_embedding()
 
     def knowledge_embedding_batch(self, docs):
+        """Deprecation"""
         # docs = self.knowledge_embedding_client.read_batch()
         return self.knowledge_embedding_client.index_to_store(docs)
 
     def read(self):
+        """Deprecation"""
         self.knowledge_embedding_client = self.init_knowledge_embedding()
         return self.knowledge_embedding_client.read_batch()
 

pilot/model/adapter.py

@@ -73,6 +73,7 @@ class VicunaLLMAdapater(BaseLLMAdaper):
         )
         return model, tokenizer
 
+
 def auto_configure_device_map(num_gpus):
     """handling multi gpu calls"""
     # transformer.word_embeddings occupying 1 floors
@@ -88,11 +89,11 @@ def auto_configure_device_map(num_gpus):
     # If transformer. word_ If embeddings. device and model. device are different, it will cause a RuntimeError
     # Therefore, here we will transform. word_ Embeddings, transformer. final_ Layernorm, lm_ Put all the heads on the first card
     device_map = {
-        'transformer.embedding.word_embeddings': 0,
+        "transformer.embedding.word_embeddings": 0,
-        'transformer.encoder.final_layernorm': 0,
+        "transformer.encoder.final_layernorm": 0,
-        'transformer.output_layer': 0,
+        "transformer.output_layer": 0,
-        'transformer.rotary_pos_emb': 0,
+        "transformer.rotary_pos_emb": 0,
-        'lm_head': 0
+        "lm_head": 0,
     }
 
     used = 2
@@ -102,7 +103,7 @@ def auto_configure_device_map(num_gpus):
             gpu_target += 1
             used = 0
         assert gpu_target < num_gpus
-        device_map[f'transformer.encoder.layers.{i}'] = gpu_target
+        device_map[f"transformer.encoder.layers.{i}"] = gpu_target
         used += 1
 
     return device_map
@@ -114,7 +115,13 @@ class ChatGLMAdapater(BaseLLMAdaper):
     def match(self, model_path: str):
         return "chatglm" in model_path
 
-    def loader(self, model_path: str, from_pretrained_kwargs: dict, device_map=None, num_gpus=CFG.NUM_GPUS):
+    def loader(
+        self,
+        model_path: str,
+        from_pretrained_kwargs: dict,
+        device_map=None,
+        num_gpus=CFG.NUM_GPUS,
+    ):
         tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 
         if DEVICE != "cuda":
@@ -125,10 +132,8 @@ class ChatGLMAdapater(BaseLLMAdaper):
         else:
             model = (
                 AutoModel.from_pretrained(
-                    model_path, trust_remote_code=True,
+                    model_path, trust_remote_code=True, **from_pretrained_kwargs
-                    **from_pretrained_kwargs
+                ).half()
-                )
-                .half()
                 # .cuda()
             )
             from accelerate import dispatch_model

pilot/vector_store/connector.py

@@ -6,7 +6,13 @@ connector = {"Chroma": ChromaStore, "Milvus": MilvusStore}
 
 
 class VectorStoreConnector:
-    """vector store connector, can connect different vector db provided load document api_v1 and similar search api_v1."""
+    """VectorStoreConnector, can connect different vector db provided load document api_v1 and similar search api_v1.
+    1.load_document:knowledge document source into vector store.(Chroma, Milvus, Weaviate)
+    2.similar_search: similarity search from vector_store
+    how to use reference:https://db-gpt.readthedocs.io/en/latest/modules/vector.html
+    how to integrate:https://db-gpt.readthedocs.io/en/latest/modules/vector/milvus/milvus.html
+
+    """
 
     def __init__(self, vector_store_type, ctx: {}) -> None:
         """initialize vector store connector."""