github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-15 05:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Support i18n (#1327)

Browse Source
This commit is contained in:
Fangyin Cheng
2024-03-25 20:15:39 +08:00
committed by GitHub
parent fa06be64c1
commit fcc325d411
179 changed files with 12052 additions and 69512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
dbgpt/storage/vector_store/base.py
View File

@@ -8,10 +8,81 @@ from typing import List, Optional
from dbgpt._private.pydantic import BaseModel, Field
from dbgpt.core import Chunk, Embeddings
from dbgpt.core.awel.flow import Parameter
from dbgpt.util.i18n_utils import _
logger = logging.getLogger(__name__)
_COMMON_PARAMETERS = [
Parameter.build_from(
_("Collection Name"),
"name",
str,
description=_(
"The name of vector store, if not set, will use the default " "name."
),
optional=True,
default="dbgpt_collection",
),
Parameter.build_from(
_("User"),
"user",
str,
description=_(
"The user of vector store, if not set, will use the default " "user."
),
optional=True,
default=None,
),
Parameter.build_from(
_("Password"),
"password",
str,
description=_(
"The password of vector store, if not set, will use the "
"default password."
),
optional=True,
default=None,
),
Parameter.build_from(
_("Embedding Function"),
"embedding_fn",
Embeddings,
description=_(
"The embedding function of vector store, if not set, will use "
"the default embedding function."
),
optional=True,
default=None,
),
Parameter.build_from(
_("Max Chunks Once Load"),
"max_chunks_once_load",
int,
description=_(
"The max number of chunks to load at once. If your document is "
"large, you can set this value to a larger number to speed up the loading "
"process. Default is 10."
),
optional=True,
default=10,
),
Parameter.build_from(
_("Max Threads"),
"max_threads",
int,
description=_(
"The max number of threads to use. Default is 1. If you set "
"this bigger than 1, please make sure your vector store is thread-safe."
),
optional=True,
default=1,
),
]
class VectorStoreConfig(BaseModel):
"""Vector store config."""

25
dbgpt/storage/vector_store/chroma_store.py
View File

@@ -8,15 +8,32 @@ from chromadb.config import Settings
from dbgpt._private.pydantic import Field
from dbgpt.configs.model_config import PILOT_PATH
# TODO: Recycle dependency on rag and storage
from dbgpt.core import Chunk
from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource
from dbgpt.util.i18n_utils import _
from .base import VectorStoreBase, VectorStoreConfig
from .base import _COMMON_PARAMETERS, VectorStoreBase, VectorStoreConfig
logger = logging.getLogger(__name__)
@register_resource(
_("Chroma Vector Store"),
"chroma_vector_store",
category=ResourceCategory.VECTOR_STORE,
description=_("Chroma vector store."),
parameters=[
*_COMMON_PARAMETERS,
Parameter.build_from(
_("Persist Path"),
"persist_path",
str,
description=_("the persist path of vector store."),
optional=True,
default=None,
),
],
)
class ChromaVectorConfig(VectorStoreConfig):
"""Chroma vector store config."""
@@ -43,7 +60,7 @@ class ChromaStore(VectorStoreBase):
"""Create a ChromaStore instance."""
from langchain.vectorstores import Chroma
chroma_vector_config = vector_store_config.dict()
chroma_vector_config = vector_store_config.dict(exclude_none=True)
chroma_path = chroma_vector_config.get(
"persist_path", os.path.join(PILOT_PATH, "data")
)

38
dbgpt/storage/vector_store/connector.py
View File

@@ -4,12 +4,50 @@ import os
from typing import Any, Dict, List, Optional, Type, cast
from dbgpt.core import Chunk
from dbgpt.core.awel.flow import (
FunctionDynamicOptions,
OptionValue,
Parameter,
ResourceCategory,
register_resource,
)
from dbgpt.storage import vector_store
from dbgpt.storage.vector_store.base import VectorStoreBase, VectorStoreConfig
from dbgpt.util.i18n_utils import _
connector: Dict[str, Type] = {}
def _load_vector_options() -> List[OptionValue]:
return [
OptionValue(label=cls, name=cls, value=cls)
for cls in vector_store.__all__
if issubclass(getattr(vector_store, cls), VectorStoreBase)
]
@register_resource(
_("Vector Store Connector"),
"vector_store_connector",
category=ResourceCategory.VECTOR_STORE,
parameters=[
Parameter.build_from(
_("Vector Store Type"),
"vector_store_type",
str,
description=_("The type of vector store."),
options=FunctionDynamicOptions(func=_load_vector_options),
),
Parameter.build_from(
_("Vector Store Implementation"),
"vector_store_config",
VectorStoreConfig,
description=_("The vector store implementation."),
optional=True,
default=None,
),
],
)
class VectorStoreConnector:
"""The connector for vector store.

89
dbgpt/storage/vector_store/milvus_store.py
View File

@@ -8,12 +8,90 @@ from typing import Any, Iterable, List, Optional
from dbgpt._private.pydantic import Field
from dbgpt.core import Chunk, Embeddings
from dbgpt.storage.vector_store.base import VectorStoreBase, VectorStoreConfig
from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource
from dbgpt.storage.vector_store.base import (
_COMMON_PARAMETERS,
VectorStoreBase,
VectorStoreConfig,
)
from dbgpt.util import string_utils
from dbgpt.util.i18n_utils import _
logger = logging.getLogger(__name__)
@register_resource(
_("Milvus Vector Store"),
"milvus_vector_store",
category=ResourceCategory.VECTOR_STORE,
parameters=[
*_COMMON_PARAMETERS,
Parameter.build_from(
_("Uri"),
"uri",
str,
description=_(
"The uri of milvus store, if not set, will use the default " "uri."
),
optional=True,
default="localhost",
),
Parameter.build_from(
_("Port"),
"port",
str,
description=_(
"The port of milvus store, if not set, will use the default " "port."
),
optional=True,
default="19530",
),
Parameter.build_from(
_("Alias"),
"alias",
str,
description=_(
"The alias of milvus store, if not set, will use the default " "alias."
),
optional=True,
default="default",
),
Parameter.build_from(
_("Primary Field"),
"primary_field",
str,
description=_(
"The primary field of milvus store, if not set, will use the "
"default primary field."
),
optional=True,
default="pk_id",
),
Parameter.build_from(
_("Text Field"),
"text_field",
str,
description=_(
"The text field of milvus store, if not set, will use the "
"default text field."
),
optional=True,
default="content",
),
Parameter.build_from(
_("Embedding Field"),
"embedding_field",
str,
description=_(
"The embedding field of milvus store, if not set, will use the "
"default embedding field."
),
optional=True,
default="vector",
),
],
description=_("Milvus vector store."),
)
class MilvusVectorConfig(VectorStoreConfig):
"""Milvus vector store config."""
@@ -36,15 +114,6 @@ class MilvusVectorConfig(VectorStoreConfig):
description="The alias of milvus store, if not set, will use the default "
"alias.",
)
user: str = Field(
default=None,
description="The user of milvus store, if not set, will use the default user.",
)
password: str = Field(
default=None,
description="The password of milvus store, if not set, will use the default "
"password.",
)
primary_field: str = Field(
default="pk_id",
description="The primary field of milvus store, if not set, will use the "

31
dbgpt/storage/vector_store/pgvector_store.py
View File

@@ -2,16 +2,39 @@
import logging
from typing import Any, List
from dbgpt._private.config import Config
from dbgpt._private.pydantic import Field
from dbgpt.core import Chunk
from dbgpt.storage.vector_store.base import VectorStoreBase, VectorStoreConfig
from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource
from dbgpt.storage.vector_store.base import (
_COMMON_PARAMETERS,
VectorStoreBase,
VectorStoreConfig,
)
from dbgpt.util.i18n_utils import _
logger = logging.getLogger(__name__)
CFG = Config()
@register_resource(
_("PG Vector Store"),
"pg_vector_store",
category=ResourceCategory.VECTOR_STORE,
parameters=[
*_COMMON_PARAMETERS,
Parameter.build_from(
_("Connection String"),
"connection_string",
str,
description=_(
"The connection string of vector store, if not set, will use "
"the default connection string."
),
optional=True,
default=None,
),
],
description="PG vector store.",
)
class PGVectorConfig(VectorStoreConfig):
"""PG vector store config."""

33
dbgpt/storage/vector_store/weaviate_store.py
View File

@@ -3,16 +3,43 @@ import logging
import os
from typing import List
from dbgpt._private.config import Config
from dbgpt._private.pydantic import Field
from dbgpt.core import Chunk
from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource
from dbgpt.util.i18n_utils import _
from .base import VectorStoreBase, VectorStoreConfig
from .base import _COMMON_PARAMETERS, VectorStoreBase, VectorStoreConfig
logger = logging.getLogger(__name__)
CFG = Config()
@register_resource(
_("Weaviate Vector Store"),
"weaviate_vector_store",
category=ResourceCategory.VECTOR_STORE,
description=_("Weaviate vector store."),
parameters=[
*_COMMON_PARAMETERS,
Parameter.build_from(
_("Weaviate URL"),
"weaviate_url",
str,
description=_(
"weaviate url address, if not set, will use the default url."
),
optional=True,
default=None,
),
Parameter.build_from(
_("Persist Path"),
"persist_path",
str,
description=_("the persist path of vector store."),
optional=True,
default=None,
),
],
)
class WeaviateVectorConfig(VectorStoreConfig):
"""Weaviate vector store config."""