github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-18 08:17:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

update:milvus_batch size

Browse Source
This commit is contained in:
aries-ckt 2023-05-25 17:42:42 +08:00
parent 6c964c8a3d
commit 723243d0cd
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pilot/vector_store/milvus_store.py
View File

@ -133,7 +133,7 @@ class MilvusStore(VectorStoreBase):
max_length = max(max_length, len(y)) max_length = max(max_length, len(y))
# Create the text field # Create the text field
fields.append( fields.append(
FieldSchema(text_field, DataType.VARCHAR, max_length=max_length + 100) FieldSchema(text_field, DataType.VARCHAR, max_length=max_length + 1)
) )
# primary key field # primary key field
fields.append( fields.append(
@ -250,11 +250,12 @@ class MilvusStore(VectorStoreBase):
def load_document(self, documents) -> None: def load_document(self, documents) -> None:
"""load document in vector database.""" """load document in vector database."""
batch_size = 500 self.init_schema_and_load(self.collection_name, documents)
batched_list = [documents[i:i + batch_size] for i in range(0, len(documents), batch_size)] # batch_size = 500
# docs = [] # batched_list = [documents[i:i + batch_size] for i in range(0, len(documents), batch_size)]
for doc_batch in batched_list: # # docs = []
self.init_schema_and_load(self.collection_name, doc_batch) # for doc_batch in batched_list:
# self.init_schema_and_load(self.collection_name, doc_batch)
def similar_search(self, text, topk) -> None: def similar_search(self, text, topk) -> None:
"""similar_search in vector database.""" """similar_search in vector database."""