Merge branch 'main' into feat_rag_retrieve_mode

This commit is contained in:
alan.cl 2025-07-14 21:24:12 +08:00
commit 8d1c88d33e
24 changed files with 577 additions and 49 deletions

View File

@ -1,12 +1,12 @@
# Oceanbase Vector RAG
# OceanBase Vector RAG
In this example, we will show how to use the Oceanbase Vector as in DB-GPT RAG Storage. Using a graph database to implement RAG can, to some extent, alleviate the uncertainty and interpretability issues brought about by vector database retrieval.
In this example, we will show how to use the OceanBase Vector as in DB-GPT RAG Storage. Using a graph database to implement RAG can, to some extent, alleviate the uncertainty and interpretability issues brought about by vector database retrieval.
### Install Dependencies
First, you need to install the `dbgpt Oceanbase Vector storage` library.
First, you need to install the `dbgpt OceanBase Vector storage` library.
```bash
uv sync --all-packages \
@ -17,19 +17,19 @@ uv sync --all-packages \
--extra "dbgpts"
````
### Prepare Oceanbase Vector
### Prepare OceanBase Vector
Prepare Oceanbase Vector database service, reference[Oceanbase Vector](https://open.oceanbase.com/) .
Prepare OceanBase Vector database service, reference[OceanBase Vector](https://open.oceanbase.com/) .
### TuGraph Configuration
### OceanBase Configuration
Set rag storage variables below in `configs/dbgpt-proxy-openai.toml` file, let DB-GPT know how to connect to Oceanbase Vector.
Set rag storage variables below in `configs/dbgpt-proxy-openai.toml` file, let DB-GPT know how to connect to OceanBase Vector.
```
[rag.storage]
[rag.storage.vector]
type = "Oceanbase"
type = "oceanbase"
uri = "127.0.0.1"
port = "19530"
#username="dbgpt"

View File

@ -22,8 +22,12 @@ async def main():
# initialize client
DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
res = await list_app(client)
print(res)
try:
res = await list_app(client)
print(res)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
if __name__ == "__main__":

View File

@ -57,12 +57,16 @@ async def main():
# initialize client
DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
data = await client.chat(model="Qwen2.5-72B-Instruct", messages="hello")
try:
data = await client.chat(model="Qwen2.5-72B-Instruct", messages="hello")
print(data)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
# async for data in client.chat_stream(
# model="chatgpt_proxyllm",
# messages="hello",
# ):
print(data)
# res = await client.chat(model="chatgpt_proxyllm" ,messages="hello")
# print(res)

View File

@ -83,8 +83,12 @@ async def main():
}
],
)
data = await run_evaluation(client, request=request)
print(data)
try:
data = await run_evaluation(client, request=request)
print(data)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
if __name__ == "__main__":

View File

@ -55,8 +55,12 @@ async def main():
# initialize client
DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
res = await list_datasource(client)
print(res)
try:
res = await list_datasource(client)
print(res)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
if __name__ == "__main__":

View File

@ -37,8 +37,12 @@ async def main():
# initialize client
DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
res = await list_flow(client)
print(res)
try:
res = await list_flow(client)
print(res)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
if __name__ == "__main__":

View File

@ -75,16 +75,20 @@ async def main():
DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
res = await create_space(
client,
SpaceModel(
name="test_space_1",
vector_type="Chroma",
desc="for client space desc",
owner="dbgpt",
),
)
print(res)
try:
res = await create_space(
client,
SpaceModel(
name="test_space_1",
vector_type="Chroma",
desc="for client space desc",
owner="dbgpt",
),
)
print(res)
finally:
# explicitly close client to avoid event loop closed error
await client.aclose()
# list all spaces
# res = await list_space(client)

View File

@ -0,0 +1 @@
{"pageProps":{"type":"add"},"__N_SSG":true}

View File

@ -0,0 +1 @@
{"pageProps":{"type":"edit"},"__N_SSG":true}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1 @@
self.__SSG_MANIFEST=new Set(["\u002Fconstruct\u002Fprompt\u002F[type]"]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,344 @@
"""GaussDB connector."""
import logging
from dataclasses import dataclass, field
from typing import Any, Iterable, List, Optional, Tuple, Type, cast
from urllib.parse import quote
from urllib.parse import quote_plus as urlquote
from sqlalchemy import text
from dbgpt.core.awel.flow import (
TAGS_ORDER_HIGH,
ResourceCategory,
auto_register_resource,
)
from dbgpt.datasource.rdbms.base import RDBMSConnector, RDBMSDatasourceParameters
from dbgpt.util.i18n_utils import _
logger = logging.getLogger(__name__)
@auto_register_resource(
label=_("GaussDB datasource"),
category=ResourceCategory.DATABASE,
tags={"order": TAGS_ORDER_HIGH},
description=_(
"A scalable,enterprise-grade relational database with distributed architecture."
),
)
@dataclass
class GaussDBParameters(RDBMSDatasourceParameters):
"""GaussDB connection parameters."""
__type__ = "gaussdb"
schema: str = field(
default="public", metadata={"help": _("Database schema, defaults to 'public'")}
)
driver: str = field(
default="postgresql+psycopg2",
metadata={
"help": _("Driver name for postgres, default is postgresql+psycopg2."),
},
)
def create_connector(self) -> "GaussDBConnector":
"""Create GaussDB connector."""
return GaussDBConnector.from_parameters(self)
def db_url(self, ssl: bool = False, charset: Optional[str] = None) -> str:
return f"{self.driver}://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
class GaussDBConnector(RDBMSConnector):
"""GaussDBConnector connector."""
driver = "postgresql+psycopg2"
db_type = "gaussdb"
db_dialect = "gaussdb"
@classmethod
def param_class(cls) -> Type[GaussDBParameters]:
"""Return the parameter class."""
return GaussDBParameters
@classmethod
def from_uri_db(
cls,
host: str,
port: int,
user: str,
pwd: str,
db_name: str,
engine_args: Optional[dict] = None,
**kwargs: Any,
) -> "GaussDBConnector":
"""Create a new PostgreSQLConnector from host, port, user, pwd, db_name."""
db_url: str = (
f"{cls.driver}://{quote(user)}:{urlquote(pwd)}@{host}:{str(port)}/{db_name}"
)
from sqlalchemy.dialects.postgresql.base import PGDialect
PGDialect._get_server_version_info = lambda *args: (7, 0)
return cast(GaussDBConnector, cls.from_uri(db_url, engine_args, **kwargs))
@classmethod
def from_parameters(cls, parameters: GaussDBParameters) -> "RDBMSConnector":
"""Create a new connector from parameters."""
return cls.from_uri_db(
parameters.host,
parameters.port,
parameters.user,
parameters.password,
parameters.database,
schema=parameters.schema,
engine_args=parameters.engine_args(),
)
def _sync_tables_from_db(self) -> Iterable[str]:
"""Read table information from database with schema support."""
schema = self._schema or "public"
with self.session_scope() as session:
# Get tables for specific schema
table_results = session.execute(
text(
"""
SELECT tablename
FROM pg_catalog.pg_tables
WHERE schemaname = :schema
"""
),
{"schema": schema},
)
# Get views for specific schema
view_results = session.execute(
text(
"""
SELECT viewname
FROM pg_catalog.pg_views
WHERE schemaname = :schema
"""
),
{"schema": schema},
)
table_results = set(row[0] for row in table_results)
view_results = set(row[0] for row in view_results)
self._all_tables = table_results.union(view_results)
# Reflect with schema
self._metadata.reflect(bind=self._engine, schema=schema)
return self._all_tables
def get_grants(self):
"""Get grants."""
with self.session_scope() as session:
cursor = session.execute(
text(
"""
SELECT DISTINCT grantee, privilege_type
FROM information_schema.role_table_grants
WHERE grantee = CURRENT_USER;"""
)
)
grants = cursor.fetchall()
return grants
def get_collation(self):
"""Get collation."""
try:
with self.session_scope() as session:
cursor = session.execute(
text(
"SELECT datcollate AS collation FROM pg_database WHERE "
"datname = current_database();"
)
)
collation = cursor.fetchone()[0]
return collation
except Exception as e:
logger.warning(f"postgresql get collation error: {str(e)}")
return None
def get_users(self):
"""Get user info."""
try:
with self.session_scope() as session:
cursor = session.execute(
text("SELECT rolname FROM pg_roles WHERE rolname NOT LIKE 'pg_%';")
)
users = cursor.fetchall()
return [user[0] for user in users]
except Exception as e:
logger.warning(f"postgresql get users error: {str(e)}")
return []
def get_fields(self, table_name: str, db_name: Optional[str] = None) -> List[Tuple]:
"""Get column fields about specified table."""
schema = self._schema or "public"
sql = """
SELECT
column_name,
data_type,
column_default,
is_nullable,
col_description(
(quote_ident(:schema) || '.' || quote_ident(:table))::regclass::oid,
ordinal_position
) as column_comment
FROM information_schema.columns
WHERE table_schema = :schema
AND table_name = :table
ORDER BY ordinal_position
"""
with self.session_scope() as session:
cursor = session.execute(
text(sql),
{"schema": schema, "table": table_name},
)
fields = cursor.fetchall()
return [
(field[0], field[1], field[2], field[3], field[4]) for field in fields
]
def get_charset(self):
"""Get character_set."""
with self.session_scope() as session:
cursor = session.execute(
text(
"SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE "
"datname = current_database();"
)
)
character_set = cursor.fetchone()[0]
return character_set
def get_show_create_table(self, table_name: str) -> str:
"""Return show create table with schema support."""
schema = self._schema or "public"
with self.session_scope() as session:
# Get column definitions
cur = session.execute(
text(
"""
SELECT
column_name,
data_type,
column_default,
is_nullable,
character_maximum_length,
numeric_precision,
numeric_scale
FROM information_schema.columns
WHERE table_schema = :schema
AND table_name = :table
ORDER BY ordinal_position
"""
),
{"schema": schema, "table": table_name},
)
create_table = f"CREATE TABLE {schema}.{table_name} (\n"
for row in cur.fetchall():
col_name = row[0]
data_type = row[1]
default = f"DEFAULT {row[2]}" if row[2] else ""
nullable = "NOT NULL" if row[3] == "NO" else ""
# Add length/precision/scale if applicable
if row[4]: # character_maximum_length
data_type = f"{data_type}({row[4]})"
elif row[5]: # numeric_precision
if row[6]: # numeric_scale
data_type = f"{data_type}({row[5]},{row[6]})"
else:
data_type = f"{data_type}({row[5]})"
create_table += f" {col_name} {data_type} {default} {nullable},\n"
create_table = create_table.rstrip(",\n") + "\n)"
return create_table
def get_table_comments(self, db_name=None):
"""Get table comments."""
tablses = self.table_simple_info()
comments = []
for table in tablses:
table_name = table[0]
table_comment = self.get_show_create_table(table_name)
comments.append((table_name, table_comment))
return comments
def get_database_names(self):
"""Get database names."""
with self.session_scope() as session:
cursor = session.execute(text("SELECT datname FROM pg_database;"))
results = cursor.fetchall()
return [
d[0]
for d in results
if d[0] not in ["template0", "template1", "postgres"]
]
def get_current_db_name(self) -> str:
"""Get current database name."""
with self.session_scope() as session:
return session.execute(text("SELECT current_database()")).scalar()
def table_simple_info(self):
"""Get table simple info."""
_sql = """
SELECT table_name, string_agg(column_name, ', ') AS schema_info
FROM (
SELECT c.relname AS table_name, a.attname AS column_name
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid
WHERE c.relkind = 'r'
AND a.attnum > 0
AND NOT a.attisdropped
AND n.nspname NOT LIKE 'pg_%'
AND n.nspname != 'information_schema'
ORDER BY c.relname, a.attnum
) sub
GROUP BY table_name;
"""
with self.session_scope() as session:
cursor = session.execute(text(_sql))
results = cursor.fetchall()
results_str = []
for result in results:
results_str.append((str(result[0]), str(result[1])))
return results_str
def get_fields_wit_schema(self, table_name, schema_name="public"):
"""Get column fields about specified table."""
query_sql = f"""
SELECT c.column_name, c.data_type, c.column_default, c.is_nullable,
d.description FROM information_schema.columns c
LEFT JOIN pg_catalog.pg_description d
ON (c.table_schema || '.' || c.table_name)::regclass::oid = d.objoid
AND c.ordinal_position = d.objsubid
WHERE c.table_name='{table_name}' AND c.table_schema='{schema_name}'
"""
with self.session_scope() as session:
cursor = session.execute(text(query_sql))
fields = cursor.fetchall()
return [
(field[0], field[1], field[2], field[3], field[4]) for field in fields
]
def get_indexes(self, table_name):
"""Get table indexes about specified table."""
with self.session_scope() as session:
cursor = session.execute(
text(
f"SELECT indexname, indexdef FROM pg_indexes WHERE "
f"tablename = '{table_name}'"
)
)
indexes = cursor.fetchall()
return [(index[0], index[1]) for index in indexes]

View File

@ -24,6 +24,7 @@ class DBType(Enum):
Oracle = DbInfo("oracle")
MSSQL = DbInfo("mssql")
Postgresql = DbInfo("postgresql")
GaussDB = DbInfo("gaussdb")
Vertica = DbInfo("vertica")
Clickhouse = DbInfo("clickhouse")
StarRocks = DbInfo("starrocks")

View File

@ -204,7 +204,7 @@ class OceanBaseStore(VectorStoreBase):
"Please install it with `pip install pyobvector`."
)
if vector_store_config.embedding_fn is None:
if embedding_fn is None:
raise ValueError("embedding_fn is required for OceanBaseStore")
super().__init__(

View File

@ -53,6 +53,9 @@ class ConnectorManager(BaseComponent):
)
from dbgpt_ext.datasource.rdbms.conn_doris import DorisConnector # noqa: F401
from dbgpt_ext.datasource.rdbms.conn_duckdb import DuckDbConnector # noqa: F401
from dbgpt_ext.datasource.rdbms.conn_gaussdb import ( # noqa: F401
GaussDBConnector,
)
from dbgpt_ext.datasource.rdbms.conn_hive import HiveConnector # noqa: F401
from dbgpt_ext.datasource.rdbms.conn_mssql import MSSQLConnector # noqa: F401
from dbgpt_ext.datasource.rdbms.conn_mysql import MySQLConnector # noqa: F401
@ -197,8 +200,22 @@ class ConnectorManager(BaseComponent):
db_port = db_config.get("db_port")
db_user = db_config.get("db_user")
db_pwd = db_config.get("db_pwd")
try:
ext_config = db_config.get("ext_config")
db_json = json.loads(ext_config)
schema = db_json.get("schema", None)
except json.JSONDecodeError:
# 处理解码失败的情况
db_json = {}
schema = None
return connect_instance.from_uri_db( # type: ignore
host=db_host, port=db_port, user=db_user, pwd=db_pwd, db_name=db_name
host=db_host,
port=db_port,
user=db_user,
pwd=db_pwd,
db_name=db_name,
schema=schema,
)
def _create_parameters(

View File

@ -135,7 +135,7 @@ const PromptBot: React.FC<PromptBotProps> = ({ submit, chat_scene }) => {
onOpenChange={handleOpenChange}
>
<Tooltip title={t('Click_Select') + ' Prompt'}>
<FloatButton className='bottom-[30%]' />
<FloatButton className='right-4 md:right-6 bottom-[180px] md:bottom-[160px] z-[998]' />
</Tooltip>
</Popover>
</ConfigProvider>

View File

@ -79,9 +79,7 @@ const ChatContentContainer = ({ className }: { className?: string }, ref: React.
currentScrollRef.removeEventListener('scroll', handleScroll);
}
};
}, [handleScroll]);
const scrollToBottomSmooth = useCallback((forceScroll = false) => {
}, [handleScroll]); const scrollToBottomSmooth = useCallback((forceScroll = false) => {
if (!scrollRef.current) return;
// For force scroll (new messages), bypass allowAutoScroll check
@ -119,9 +117,7 @@ const ChatContentContainer = ({ className }: { className?: string }, ref: React.
const lastMessage = useMemo(() => {
const last = history[history.length - 1];
return last ? { context: last.context, thinking: last.thinking } : null;
}, [history]);
// Track previous history length to detect new messages
}, [history]); // Track previous history length to detect new messages
const prevHistoryLengthRef = useRef(history.length);
useEffect(() => {
@ -153,9 +149,7 @@ const ChatContentContainer = ({ className }: { className?: string }, ref: React.
cancelAnimationFrame(animationFrameRef.current);
}
};
}, []);
const scrollToTop = useCallback(() => {
}, []); const scrollToTop = useCallback(() => {
if (scrollRef.current) {
scrollRef.current.scrollTo({
top: 0,
@ -181,23 +175,23 @@ const ChatContentContainer = ({ className }: { className?: string }, ref: React.
</div>
{showScrollButtons && (
<div className='absolute right-6 bottom-24 flex flex-col gap-2'>
<div className='absolute right-4 md:right-6 bottom-[120px] md:bottom-[100px] flex flex-col gap-2 z-[999]'>
{!isAtTop && (
<button
onClick={scrollToTop}
className='w-10 h-10 bg-white dark:bg-[rgba(255,255,255,0.2)] border border-gray-200 dark:border-[rgba(255,255,255,0.2)] rounded-full flex items-center justify-center shadow-md hover:shadow-lg transition-shadow'
className='w-9 h-9 md:w-10 md:h-10 bg-white dark:bg-[rgba(255,255,255,0.2)] border border-gray-200 dark:border-[rgba(255,255,255,0.2)] rounded-full flex items-center justify-center shadow-md hover:shadow-lg transition-all duration-200'
aria-label='Scroll to top'
>
<VerticalAlignTopOutlined className='text-[#525964] dark:text-[rgba(255,255,255,0.85)]' />
<VerticalAlignTopOutlined className='text-[#525964] dark:text-[rgba(255,255,255,0.85)] text-sm md:text-base' />
</button>
)}
{!isAtBottom && (
<button
onClick={scrollToBottom}
className='w-10 h-10 bg-white dark:bg-[rgba(255,255,255,0.2)] border border-gray-200 dark:border-[rgba(255,255,255,0.2)] rounded-full flex items-center justify-center shadow-md hover:shadow-lg transition-shadow'
className='w-9 h-9 md:w-10 md:h-10 bg-white dark:bg-[rgba(255,255,255,0.2)] border border-gray-200 dark:border-[rgba(255,255,255,0.2)] rounded-full flex items-center justify-center shadow-md hover:shadow-lg transition-all duration-200'
aria-label='Scroll to bottom'
>
<VerticalAlignBottomOutlined className='text-[#525964] dark:text-[rgba(255,255,255,0.85)]' />
<VerticalAlignBottomOutlined className='text-[#525964] dark:text-[rgba(255,255,255,0.85)] text-sm md:text-base' />
</button>
)}
</div>
@ -206,4 +200,4 @@ const ChatContentContainer = ({ className }: { className?: string }, ref: React.
);
};
export default forwardRef(ChatContentContainer);
export default forwardRef(ChatContentContainer);

View File

@ -4,9 +4,12 @@ import React from 'react';
const FloatHelper: React.FC = () => {
return (
<FloatButton.Group trigger='hover' icon={<SmileOutlined />}>
<FloatButton icon={<ReadOutlined />} href='http://docs.dbgpt.cn' target='_blank' tooltip='Doucuments' />
</FloatButton.Group>
<div className='fixed right-4 md:right-6 bottom-[240px] md:bottom-[220px] z-[997]'>
<FloatButton.Group trigger='hover' icon={<SmileOutlined />}>
<FloatButton icon={<ReadOutlined />} href='http://docs.dbgpt.cn' target='_blank' tooltip='Documents' />
</FloatButton.Group>
</div>
);
};
export default FloatHelper;

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

View File

@ -112,6 +112,44 @@ pre {
padding-left: 0.5rem;
}
/* Floating buttons z-index system */
.z-float-helper {
z-index: 997;
}
.z-prompt-bot {
z-index: 998;
}
.z-scroll-buttons {
z-index: 999;
}
/* Responsive floating button layout */
@media (max-width: 768px) {
.float-button-container {
right: 1rem !important;
gap: 0.5rem;
}
.float-button-container button {
width: 2.25rem !important;
height: 2.25rem !important;
}
}
@media (min-width: 769px) {
.float-button-container {
right: 1.5rem !important;
gap: 0.5rem;
}
.float-button-container button {
width: 2.5rem !important;
height: 2.5rem !important;
}
}
table {
display: block;
width: 100%;

View File

@ -295,6 +295,11 @@ export const dbMapper: Record<DBType, { label: string; icon: string; desc: strin
icon: '/icons/postgresql.png',
desc: 'Powerful open-source relational database with extensibility and SQL standards.',
},
gaussdb: {
label: 'GaussDB',
icon: '/icons/gaussdb.png',
desc: 'Huawei\'s distributed database with PostgreSQL compatibility',
},
vertica: {
label: 'Vertica',
icon: '/icons/vertica.png',