diff --git a/pilot/embedding_engine/url_embedding.py b/pilot/embedding_engine/url_embedding.py index 113e2985e..ce9e5360d 100644 --- a/pilot/embedding_engine/url_embedding.py +++ b/pilot/embedding_engine/url_embedding.py @@ -3,7 +3,7 @@ from typing import List from bs4 import BeautifulSoup from langchain.document_loaders import WebBaseLoader from langchain.schema import Document -from langchain.text_splitter import CharacterTextSplitter +from langchain.text_splitter import CharacterTextSplitter, SpacyTextSplitter from pilot.configs.config import Config from pilot.configs.model_config import KNOWLEDGE_CHUNK_SPLIT_SIZE @@ -33,7 +33,11 @@ class URLEmbedding(SourceEmbedding): length_function=len, ) else: - text_splitter = CHNDocumentSplitter(pdf=True, sentence_size=1000) + text_splitter = SpacyTextSplitter( + pipeline="zh_core_web_sm", + chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, + chunk_overlap=100, + ) return loader.load_and_split(text_splitter) @register diff --git a/tools/knowledge_init.py b/tools/knowledge_init.py index 66285a784..34460e398 100644 --- a/tools/knowledge_init.py +++ b/tools/knowledge_init.py @@ -3,12 +3,12 @@ import argparse import os import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) from pilot.embedding_engine.knowledge_type import KnowledgeType from pilot.openapi.knowledge.knowledge_service import KnowledgeService from pilot.openapi.knowledge.request.knowledge_request import KnowledgeSpaceRequest -sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) from pilot.configs.config import Config from pilot.configs.model_config import (