From 977a88509e4f1e501438ce3525146a54fc43fb45 Mon Sep 17 00:00:00 2001 From: aries-ckt <916701291@qq.com> Date: Wed, 24 May 2023 15:22:10 +0800 Subject: [PATCH 1/3] update:config env --- pilot/vector_store/milvus_store.py | 18 +++++------------- requirements.txt | 1 + tools/knowlege_init.py | 11 ++++++----- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pilot/vector_store/milvus_store.py b/pilot/vector_store/milvus_store.py index a61027850..8af9240e2 100644 --- a/pilot/vector_store/milvus_store.py +++ b/pilot/vector_store/milvus_store.py @@ -139,29 +139,21 @@ class MilvusStore(VectorStoreBase): fields.append( FieldSchema(text_field, DataType.VARCHAR, max_length=max_length + 1) ) - # create the primary key field + # primary key field fields.append( FieldSchema(primary_field, DataType.INT64, is_primary=True, auto_id=True) ) - # create the vector field + # vector field fields.append(FieldSchema(vector_field, DataType.FLOAT_VECTOR, dim=dim)) - # Create the schema for the collection + # milvus the schema for the collection schema = CollectionSchema(fields) # Create the collection collection = Collection(collection_name, schema) self.col = collection - # Index parameters for the collection + # index parameters for the collection index = self.index_params - # Create the index + # milvus index collection.create_index(vector_field, index) - # Create the VectorStore - # milvus = cls( - # embedding, - # kwargs.get("connection_args", {"port": 19530}), - # collection_name, - # text_field, - # ) - # Add the texts. schema = collection.schema for x in schema.fields: self.fields.append(x.name) diff --git a/requirements.txt b/requirements.txt index aea4f00e0..685661026 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,6 +69,7 @@ colorama playsound distro pypdf +milvus-cli==0.3.2 # Testing dependencies pytest diff --git a/tools/knowlege_init.py b/tools/knowlege_init.py index 23ca33a80..e64521031 100644 --- a/tools/knowlege_init.py +++ b/tools/knowlege_init.py @@ -2,11 +2,11 @@ # -*- coding: utf-8 -*- import argparse -from pilot.configs.model_config import DATASETS_DIR, LLM_MODEL_CONFIG, VECTOR_SEARCH_TOP_K, VECTOR_STORE_CONFIG, \ - VECTOR_STORE_TYPE +from pilot.configs.config import Config +from pilot.configs.model_config import DATASETS_DIR, LLM_MODEL_CONFIG, VECTOR_SEARCH_TOP_K from pilot.source_embedding.knowledge_embedding import KnowledgeEmbedding - +CFG = Config() class LocalKnowledgeInit: embeddings: object = None model_name = LLM_MODEL_CONFIG["text2vec"] @@ -32,6 +32,7 @@ class LocalKnowledgeInit: dc, s = doc yield s, dc + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--vector_name", type=str, default="default") @@ -40,8 +41,8 @@ if __name__ == "__main__": args = parser.parse_args() vector_name = args.vector_name append_mode = args.append - store_type = VECTOR_STORE_TYPE - vector_store_config = {"url": VECTOR_STORE_CONFIG["url"], "port": VECTOR_STORE_CONFIG["port"], "vector_store_name":vector_name} + store_type = CFG.VECTOR_STORE_TYPE + vector_store_config = {"vector_store_name": vector_name} print(vector_store_config) kv = LocalKnowledgeInit(vector_store_config=vector_store_config) vector_store = kv.knowledge_persist(file_path=DATASETS_DIR, append_mode=append_mode) From 94a418897a02ab9025b310e6a18242aa90be5309 Mon Sep 17 00:00:00 2001 From: aries-ckt <916701291@qq.com> Date: Wed, 24 May 2023 16:28:13 +0800 Subject: [PATCH 2/3] update:config env --- pilot/configs/config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pilot/configs/config.py b/pilot/configs/config.py index e9ec2bd48..2e933649a 100644 --- a/pilot/configs/config.py +++ b/pilot/configs/config.py @@ -38,14 +38,6 @@ class Config(metaclass=Singleton): self.use_mac_os_tts = False self.use_mac_os_tts = os.getenv("USE_MAC_OS_TTS") - # milvus or zilliz cloud configuration - self.milvus_addr = os.getenv("MILVUS_ADDR", "localhost:19530") - self.milvus_username = os.getenv("MILVUS_USERNAME") - self.milvus_password = os.getenv("MILVUS_PASSWORD") - self.milvus_collection = os.getenv("MILVUS_COLLECTION", "dbgpt") - self.milvus_secure = os.getenv("MILVUS_SECURE") == "True" - - self.authorise_key = os.getenv("AUTHORISE_COMMAND_KEY", "y") self.exit_key = os.getenv("EXIT_KEY", "n") self.image_provider = os.getenv("IMAGE_PROVIDER", True) From e596e1d6435d5fdd4dcc90ebce1a11458359e320 Mon Sep 17 00:00:00 2001 From: aries-ckt <916701291@qq.com> Date: Wed, 24 May 2023 17:26:04 +0800 Subject: [PATCH 3/3] fix:knowledge init env --- assets/DB_GPT_wechat.png | Bin 262711 -> 160456 bytes tools/knowlege_init.py | 5 +++++ 2 files changed, 5 insertions(+) diff --git a/assets/DB_GPT_wechat.png b/assets/DB_GPT_wechat.png index 8abeec70887bd4fbaf5b075d7d9d56a569e70c42..a1d7f75589f222150da31f210165ebeb1d482282 100644 GIT binary patch literal 160456 zcmeGD^;eYd_dSjy-JQ|{NO!{!4k;+ekW#`RARyf>NP~oeQc?m#4e8KbgObuPgfN73 zN!Mre@qDfC|L|S!`v+ziX0fjOy3W~W?|sf4`&>tZgzzCD1_lPn6HQex1_o9V1_llW zJ}&x`&6oBv=$8kcU=0vP
f0J
z)tHGC0B0^(($LdaL-pUa=8B(tO%n6(8u+ZcqW9m&LSR2?ioaJ%JT;5jf8PW|YW(|%
zqDk}reGR=OYyWRU{$GUrzg+nL8*nK2)3p3HCMJks;kGL1);E|2 R`F|JyX#U*pt$MRa ZG9>=Z-&9weBAuVl3>-~clUD^4U?ihh*4f+
zo@hS16=_+1^wv%L&Z$k(*~{qr_T~70nJ@akXd*K0|E=qK4fW@vzFX1IWK=uNUFx{K
z0N1hKIw6{g${u$m|r&2dl(O04D+~L`FK4?)pvLufO`vI^Y@5$`ZcIe+$-YR7A4yM37AQQtUodiJvCl4
zzj{#r@cpAVHvx>5x?n@=j^#S!UJPGE9xPsz=P~Hx5TAh)K{-2aD>r-^9FV!cxF>zJ%r#GJ0R-VT-%-zz%BbdDsxt=
z-9-Nkdk;P7-M#ZWLrbw+Cu?oX+(|nFsb{@+Nz<~wGX`Fc#NT7gKCo=yY5YUA`^*0C
zb9``CR+_9(rsFykx5F_wgRNdSEN!6w`TJ>BSC`aYfP)3ilIgiU%r;$
k?4(m(!wF#|S%uiU!O0ym)hGh!
zpXXtnhB>feb9U&|_CrC6tN46$29AWOA|MqY
LVgiSDMUe&(0M^5#eDtm*iK6M_he1
FVOwh+Mh=TFKzWQB%+9?t#IR-q&l&*J`*G!+?Zam(;n2U
z##urFGDmLGGcuMqVhqR`r!`GkPLP7D7**
z&gno&X*lNAo`u88Q33+Q=w(da)k!y|81oIIlI6oV!4y{v_Di3+H>99TsL8E$Z+y)n
zq4|%9vU$y6{yDT+3Y}%GD-
zr@5?H6{x(*$jEs5`t=i1N`~&1bbZwvi|Cr9*HtgrFAw|(^|S2_fB*7boyq54CZ9`R
z)IP(5x$r#ahuE60Y^hWE@1T*OX};n$eN&Ux&7t+B8#^XlHL_@HQ)9>@9+RLpp8Lc{
znuR7v!r%aE#P<(3-;?xfpZcg6FCESOUi4>kbMr?Pflh}$a(}uD(oXq$rF&fFe)2iB
ziMcHAGcR0uC;2ku%+zdZENMT-C+(JVh3lx~-QC@9?azAUPZSmwCTQjHez6&Ov|}&L
zZ2*yqA3q*li6y$(=