From 1868b228bcc5e4a2006b055b3441e8a10c35e3a1 Mon Sep 17 00:00:00 2001
From: aries_ckt <916701291@qq.com>
Date: Mon, 3 Jul 2023 16:09:18 +0800
Subject: [PATCH] fix:knowledge_init path 1.knowledge_init path 2.url embedding
 chunk

---
 pilot/embedding_engine/url_embedding.py | 8 ++++++--
 tools/knowledge_init.py                 | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/pilot/embedding_engine/url_embedding.py b/pilot/embedding_engine/url_embedding.py
index 113e2985e..ce9e5360d 100644
--- a/pilot/embedding_engine/url_embedding.py
+++ b/pilot/embedding_engine/url_embedding.py
@@ -3,7 +3,7 @@ from typing import List
 from bs4 import BeautifulSoup
 from langchain.document_loaders import WebBaseLoader
 from langchain.schema import Document
-from langchain.text_splitter import CharacterTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter
 
 from pilot.configs.config import Config
 from pilot.configs.model_config import KNOWLEDGE_CHUNK_SPLIT_SIZE
@@ -33,7 +33,11 @@ class URLEmbedding(SourceEmbedding):
                 length_function=len,
             )
         else:
-            text_splitter = CHNDocumentSplitter(pdf=True, sentence_size=1000)
+            text_splitter = SpacyTextSplitter(
+                pipeline="zh_core_web_sm",
+                chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
+                chunk_overlap=100,
+            )
         return loader.load_and_split(text_splitter)
 
     @register
diff --git a/tools/knowledge_init.py b/tools/knowledge_init.py
index 66285a784..34460e398 100644
--- a/tools/knowledge_init.py
+++ b/tools/knowledge_init.py
@@ -3,12 +3,12 @@
 import argparse
 import os
 import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 
 from pilot.embedding_engine.knowledge_type import KnowledgeType
 from pilot.openapi.knowledge.knowledge_service import KnowledgeService
 from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest
 
-sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 
 from pilot.configs.config import Config
 from pilot.configs.model_config import (