feat:embedding api

1.embedding_engine add source_reader param
2.docs update
3.fix chroma exit bug

pilot/embedding_engine/string_embedding.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
 from langchain.schema import Document
-from langchain.text_splitter import TextSplitter
+from langchain.text_splitter import TextSplitter, SpacyTextSplitter, RecursiveCharacterTextSplitter
 
 from pilot.embedding_engine import SourceEmbedding, register
 
@@ -13,19 +13,35 @@ class StringEmbedding(SourceEmbedding):
         self,
         file_path,
         vector_store_config,
+        source_reader: Optional = None,
         text_splitter: Optional[TextSplitter] = None,
     ):
         """Initialize raw text word path."""
-        super().__init__(file_path=file_path, vector_store_config=vector_store_config)
+        super().__init__(file_path=file_path, vector_store_config=vector_store_config, source_reader=None, text_splitter=None)
         self.file_path = file_path
         self.vector_store_config = vector_store_config
+        self.source_reader = source_reader or None
         self.text_splitter = text_splitter or None
 
     @register
     def read(self):
         """Load from String path."""
         metadata = {"source": "raw text"}
-        return [Document(page_content=self.file_path, metadata=metadata)]
+        docs = [Document(page_content=self.file_path, metadata=metadata)]
+        if self.text_splitter is None:
+            try:
+                self.text_splitter = SpacyTextSplitter(
+                    pipeline="zh_core_web_sm",
+                    chunk_size=100,
+                    chunk_overlap=100,
+                )
+            except Exception:
+                self.text_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=100, chunk_overlap=50
+                )
+
+        return self.text_splitter.split_documents(docs)
+
 
     @register
     def data_process(self, documents: List[Document]):