fix: clickhouse connect error fix (#958)

Co-authored-by: Magic <magic@B-4TMH9N3X-2120.local>
Co-authored-by: aries_ckt <916701291@qq.com>

dbgpt/app/scene/chat_db/professional_qa/chat.py

@@ -28,7 +28,6 @@ class ChatWithDbQA(BaseChat):
 
         if self.db_name:
             self.database = CFG.LOCAL_DB_MANAGE.get_connect(self.db_name)
-            self.db_connect = self.database.session
             self.tables = self.database.get_table_names()
 
         self.top_k = (

dbgpt/app/static/_next/static/01sV4-dnn1dUTis5H2TCy/_buildManifest.js

@@ -0,0 +1 @@
+self.__BUILD_MANIFEST=function(s,c,a,e,t,d,n,f,k,h,i,b){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":["static/chunks/29107295-90b90cb30c825230.js",s,c,a,d,n,f,k,"static/chunks/412-b911d4a677c64b70.js","static/chunks/981-ff77d5cc3ab95298.js","static/chunks/pages/index-d1740e3bc6dba7f5.js"],"/_error":["static/chunks/pages/_error-dee72aff9b2e2c12.js"],"/agent":[s,c,e,d,t,n,"static/chunks/pages/agent-92e9dce47267e88d.js"],"/chat":["static/chunks/pages/chat-84fbba4764166684.js"],"/chat/[scene]/[id]":["static/chunks/pages/chat/[scene]/[id]-f665336966e79cc9.js"],"/database":[s,c,a,e,t,f,h,"static/chunks/643-d8f53f40dd3c5b40.js","static/chunks/pages/database-3140f507fe61ccb8.js"],"/knowledge":[i,s,c,e,d,t,n,f,"static/chunks/551-266086fbfa0925ec.js","static/chunks/pages/knowledge-8ada4ce8fa909bf5.js"],"/knowledge/chunk":[e,t,"static/chunks/pages/knowledge/chunk-9f117a5ed799edd3.js"],"/models":[i,s,c,a,b,h,"static/chunks/pages/models-80218c46bc1d8cfa.js"],"/prompt":[s,c,a,b,"static/chunks/837-e6d4d1eb9e057050.js",k,"static/chunks/607-b224c640f6907e4b.js","static/chunks/pages/prompt-7f839dfd56bc4c20.js"],sortedPages:["/","/_app","/_error","/agent","/chat","/chat/[scene]/[id]","/database","/knowledge","/knowledge/chunk","/models","/prompt"]}}("static/chunks/64-91b49d45b9846775.js","static/chunks/479-b20198841f9a6a1e.js","static/chunks/9-bb2c54d5c06ba4bf.js","static/chunks/442-197e6cbc1e54109a.js","static/chunks/813-cce9482e33f2430c.js","static/chunks/553-df5701294eedae07.js","static/chunks/924-ba8e16df4d61ff5c.js","static/chunks/411-d9eba2657c72f766.js","static/chunks/270-2f094a936d056513.js","static/chunks/928-74244889bd7f2699.js","static/chunks/75fc9c18-a784766a129ec5fb.js","static/chunks/947-5980a3ff49069ddd.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

dbgpt/app/static/_next/static/01sV4-dnn1dUTis5H2TCy/_ssgManifest.js

@@ -0,0 +1 @@
+self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()

dbgpt/datasource/base.py

@@ -3,7 +3,7 @@
 
 """We need to design a base class.  That other connector can Write with this"""
 from abc import ABC
-from typing import Iterable, List, Optional
+from typing import Iterable, List, Optional, Any, Dict
 
 
 class BaseConnect(ABC):
@@ -50,7 +50,7 @@ class BaseConnect(ABC):
         """Get database names."""
         pass
 
-    def get_table_comments(self, db_name):
+    def get_table_comments(self, db_name: str):
         """Get table comments.
 
         Args:
@@ -58,6 +58,33 @@ class BaseConnect(ABC):
         """
         pass
 
+    def get_table_comment(self, table_name: str) -> Dict:
+        """Get table comment.
+        Args:
+            table_name (str): table name
+        Returns:
+            comment: Dict, which contains text: Optional[str], eg:["text": "comment"]
+        """
+        pass
+
+    def get_columns(self, table_name: str) -> List[Dict]:
+        """Get columns.
+        Args:
+            table_name (str): table name
+        Returns:
+            columns: List[Dict], which contains name: str, type: str, default_expression: str, is_in_primary_key: bool, comment: str
+            eg:[{'name': 'id', 'type': 'int', 'default_expression': '', 'is_in_primary_key': True, 'comment': 'id'}, ...]
+        """
+        pass
+
+    def get_column_comments(self, db_name, table_name):
+        """Get column comments.
+
+        Args:
+            table_name (_type_): _description_
+        """
+        pass
+
     def run(self, command: str, fetch: str = "all") -> List:
         """Execute sql command.
 
@@ -99,8 +126,13 @@ class BaseConnect(ABC):
         """Get the creation table sql about specified table."""
         pass
 
-    def get_indexes(self, table_name):
+    def get_indexes(self, table_name: str) -> List[Dict]:
-        """Get table indexes about specified table."""
+        """Get table indexes about specified table.
+        Args:
+            table_name (str): table name
+        Returns:
+            indexes: List[Dict], eg:[{'name': 'idx_key', 'column_names': ['id']}]
+        """
         pass
 
     @classmethod

dbgpt/datasource/manages/connect_config_db.py

@@ -197,6 +197,7 @@ class ConnectConfigDao(BaseDao):
         else:
             raise ValueError("Cannot get database by name" + db_name)
 
+        print(result)
         fields = [field[0] for field in result.cursor.description]
         row_dict = {}
         row_1 = list(result.cursor.fetchall()[0])

dbgpt/datasource/rdbms/base.py

@@ -4,7 +4,7 @@ import regex as re
 import pandas as pd
 from urllib.parse import quote
 from urllib.parse import quote_plus as urlquote
-from typing import Any, Iterable, List, Optional
+from typing import Any, Iterable, List, Optional, Dict
 import sqlalchemy
 from sqlalchemy import (
     MetaData,
@@ -227,6 +227,16 @@ class RDBMSDatabase(BaseConnect):
         final_str = "\n\n".join(tables)
         return final_str
 
+    def get_columns(self, table_name: str) -> List[Dict]:
+        """Get columns.
+        Args:
+            table_name (str): table name
+        Returns:
+            columns: List[Dict], which contains name: str, type: str, default_expression: str, is_in_primary_key: bool, comment: str
+            eg:[{'name': 'id', 'type': 'int', 'default_expression': '', 'is_in_primary_key': True, 'comment': 'id'}, ...]
+        """
+        return self._inspector.get_columns(table_name)
+
     def _get_sample_rows(self, table: Table) -> str:
         # build the select command
         command = select(table).limit(self._sample_rows_in_table_info)
@@ -292,20 +302,21 @@ class RDBMSDatabase(BaseConnect):
             query (str): SQL query to run
             fetch (str): fetch type
         """
+        result = []
+
         print(f"Query[{query}]")
         if not query:
-            return []
+            return result
         cursor = self.session.execute(text(query))
         if cursor.returns_rows:
             if fetch == "all":
                 result = cursor.fetchall()
             elif fetch == "one":
-                result = cursor.fetchone()[0]  # type: ignore
+                result = [cursor.fetchone()]
             else:
                 raise ValueError("Fetch parameter must be either 'one' or 'all'")
             field_names = tuple(i[0:] for i in cursor.keys())
 
-            result = list(result)
             result.insert(0, field_names)
             return result
 
@@ -474,12 +485,14 @@ class RDBMSDatabase(BaseConnect):
                 return token.get_real_name()
         return None
 
-    def get_indexes(self, table_name):
+    def get_indexes(self, table_name: str) -> List[Dict]:
-        """Get table indexes about specified table."""
+        """Get table indexes about specified table.
-        session = self._db_sessions()
+        Args:
-        cursor = session.execute(text(f"SHOW INDEXES FROM {table_name}"))
+            table_name:(str) table name
-        indexes = cursor.fetchall()
+        Returns:
-        return [(index[2], index[4]) for index in indexes]
+            List[Dict]:eg:[{'name': 'idx_key', 'column_names': ['id']}]
+        """
+        return self._inspector.get_indexes(table_name)
 
     def get_show_create_table(self, table_name):
         """Get table show create table about specified table."""
@@ -535,10 +548,11 @@ class RDBMSDatabase(BaseConnect):
         except Exception as e:
             return []
 
-    def get_table_comments(self, db_name):
+    def get_table_comments(self, db_name: str):
         cursor = self.session.execute(
             text(
-                f"""SELECT table_name, table_comment    FROM information_schema.tables   WHERE table_schema = '{db_name}'""".format(
+                f"""SELECT table_name, table_comment    FROM information_schema.tables  
+                    WHERE table_schema = '{db_name}'""".format(
                     db_name
                 )
             )
@@ -548,6 +562,31 @@ class RDBMSDatabase(BaseConnect):
             (table_comment[0], table_comment[1]) for table_comment in table_comments
         ]
 
+    def get_table_comment(self, table_name: str) -> Dict:
+        """Get table comments.
+
+        Args:
+            table_name (str): table name
+        Returns:
+            comment: Dict, which contains text: Optional[str], eg:["text": "comment"]
+        """
+        return self._inspector.get_table_comment(table_name)
+
+    def get_column_comments(self, db_name, table_name):
+        cursor = self.session.execute(
+            text(
+                f"""SELECT column_name, column_comment FROM information_schema.columns 
+                    WHERE table_schema = '{db_name}' and table_name = '{table_name}'
+                """.format(
+                    db_name, table_name
+                )
+            )
+        )
+        column_comments = cursor.fetchall()
+        return [
+            (column_comment[0], column_comment[1]) for column_comment in column_comments
+        ]
+
     def get_database_list(self):
         session = self._db_sessions()
         cursor = session.execute(text(" show databases;"))

dbgpt/datasource/rdbms/conn_clickhouse.py

@@ -1,10 +1,17 @@
 import re
-from typing import Optional, Any
+import sqlparse
+import clickhouse_connect
+from typing import List, Optional, Any, Iterable, Dict
 from sqlalchemy import text
 from urllib.parse import quote
+from sqlalchemy.schema import CreateTable
 from urllib.parse import quote_plus as urlquote
-
 from dbgpt.datasource.rdbms.base import RDBMSDatabase
+from clickhouse_connect.driver import httputil
+from dbgpt.storage.schema import DBType
+from sqlalchemy import (
+    MetaData,
+)
 
 
 class ClickhouseConnect(RDBMSDatabase):
@@ -20,6 +27,24 @@ class ClickhouseConnect(RDBMSDatabase):
     """db dialect"""
     db_dialect: str = "clickhouse"
 
+    client: Any = None
+
+    def __init__(self, client, **kwargs):
+        self.client = client
+
+        self._all_tables = set()
+        self.view_support = False
+        self._usable_tables = set()
+        self._include_tables = set()
+        self._ignore_tables = set()
+        self._custom_table_info = set()
+        self._indexes_in_table_info = set()
+        self._usable_tables = set()
+        self._usable_tables = set()
+        self._sample_rows_in_table_info = set()
+
+        self._metadata = MetaData()
+
     @classmethod
     def from_uri_db(
         cls,
@@ -31,21 +56,75 @@ class ClickhouseConnect(RDBMSDatabase):
         engine_args: Optional[dict] = None,
         **kwargs: Any,
     ) -> RDBMSDatabase:
-        db_url: str = (
+        big_pool_mgr = httputil.get_pool_manager(maxsize=16, num_pools=12)
-            f"{cls.driver}://{quote(user)}:{urlquote(pwd)}@{host}:{str(port)}/{db_name}"
+        client = clickhouse_connect.get_client(
+            host=host,
+            user=user,
+            password=pwd,
+            port=port,
+            connect_timeout=15,
+            database=db_name,
+            settings={"distributed_ddl_task_timeout": 300},
+            pool_mgr=big_pool_mgr,
         )
-        return cls.from_uri(db_url, engine_args, **kwargs)
 
-    def get_indexes(self, table_name):
+        cls.client = client
-        """Get table indexes about specified table."""
+        return cls(client, **kwargs)
-        return ""
+
+    @property
+    def dialect(self) -> str:
+        """Return string representation of dialect to use."""
+        pass
+
+    def get_table_names(self):
+        """Get all table names."""
+        session = self.client
+
+        with session.query_row_block_stream("SHOW TABLES") as stream:
+            tables = [row[0] for block in stream for row in block]
+            return tables
+
+    def get_indexes(self, table_name: str) -> List[Dict]:
+        """Get table indexes about specified table.
+        Args:
+            table_name (str): table name
+        Returns:
+            indexes: List[Dict], eg:[{'name': 'idx_key', 'column_names': ['id']}]
+        """
+        session = self.client
+
+        _query_sql = f""" 
+                    SELECT name AS table, primary_key, from system.tables where database ='{self.client.database}' and table = '{table_name}'
+                """
+        with session.query_row_block_stream(_query_sql) as stream:
+            indexes = [block for block in stream]
+            return [
+                {"name": "primary_key", "column_names": column_names.split(",")}
+                for table, column_names in indexes[0]
+            ]
+
+    @property
+    def table_info(self) -> str:
+        return self.get_table_info()
+
+    def get_table_info(self, table_names: Optional[List[str]] = None) -> str:
+        """Get information about specified tables.
+
+        Follows best practices as specified in: Rajkumar et al, 2022
+        (https://arxiv.org/abs/2204.00498)
+
+        If `sample_rows_in_table_info`, the specified number of sample rows will be
+        appended to each table description. This can increase performance as
+        demonstrated in the paper.
+        """
+        # TODO:
+        pass
 
     def get_show_create_table(self, table_name):
         """Get table show create table about specified table."""
-        session = self._db_sessions()
+        result = self.client.command(text(f"SHOW CREATE TABLE  {table_name}"))
-        cursor = session.execute(text(f"SHOW CREATE TABLE  {table_name}"))
+
-        ans = cursor.fetchall()
+        ans = result
-        ans = ans[0][0]
         ans = re.sub(r"\s*ENGINE\s*=\s*MergeTree\s*", " ", ans, flags=re.IGNORECASE)
         ans = re.sub(
             r"\s*DEFAULT\s*CHARSET\s*=\s*\w+\s*", " ", ans, flags=re.IGNORECASE
@@ -53,18 +132,32 @@ class ClickhouseConnect(RDBMSDatabase):
         ans = re.sub(r"\s*SETTINGS\s*\s*\w+\s*", " ", ans, flags=re.IGNORECASE)
         return ans
 
+    def get_columns(self, table_name: str) -> List[Dict]:
+        """Get columns.
+        Args:
+            table_name (str): str
+        Returns:
+            columns: List[Dict], which contains name: str, type: str, default_expression: str, is_in_primary_key: bool, comment: str
+            eg:[{'name': 'id', 'type': 'UInt64', 'default_expression': '', 'is_in_primary_key': True, 'comment': 'id'}, ...]
+        """
+        fields = self.get_fields(table_name)
+        return [
+            {"name": name, "comment": comment, "type": column_type}
+            for name, column_type, _, _, comment in fields[0]
+        ]
+
     def get_fields(self, table_name):
         """Get column fields about specified table."""
-        session = self._db_sessions()
+        session = self.client
-        cursor = session.execute(
+
-            text(
+        _query_sql = f""" 
-                f"SELECT name, type, default_expression, is_in_primary_key, comment  from system.columns where table='{table_name}'".format(
+            SELECT name, type, default_expression, is_in_primary_key, comment  from system.columns where table='{table_name}'
+        """.format(
             table_name
         )
-            )
+        with session.query_row_block_stream(_query_sql) as stream:
-        )
+            fields = [block for block in stream]
-        fields = cursor.fetchall()
+            return fields
-        return [(field[0], field[1], field[2], field[3], field[4]) for field in fields]
 
     def get_users(self):
         return []
@@ -80,31 +173,187 @@ class ClickhouseConnect(RDBMSDatabase):
         return "UTF-8"
 
     def get_database_list(self):
-        return []
+        session = self.client
+
+        with session.command("SHOW DATABASES") as stream:
+            databases = [
+                row[0]
+                for block in stream
+                for row in block
+                if row[0]
+                not in ("INFORMATION_SCHEMA", "system", "default", "information_schema")
+            ]
+            return databases
 
     def get_database_names(self):
-        return []
+        return self.get_database_list()
 
-    def get_table_comments(self, db_name):
+    def run(self, command: str, fetch: str = "all") -> List:
-        session = self._db_sessions()
+        # TODO need to be implemented
-        cursor = session.execute(
+        print("SQL:" + command)
-            text(
+        if not command or len(command) < 0:
-                f"""SELECT table, comment FROM system.tables WHERE database = '{db_name}'""".format(
+            return []
+        _, ttype, sql_type, table_name = self.__sql_parse(command)
+        if ttype == sqlparse.tokens.DML:
+            if sql_type == "SELECT":
+                return self._query(command, fetch)
+            else:
+                self._write(command)
+                select_sql = self.convert_sql_write_to_select(command)
+                print(f"write result query:{select_sql}")
+                return self._query(select_sql)
+        else:
+            print(f"DDL execution determines whether to enable through configuration ")
+
+            cursor = self.client.command(command)
+
+            if cursor.written_rows:
+                result = cursor.result_rows
+                field_names = result.column_names
+
+                result = list(result)
+                result.insert(0, field_names)
+                print("DDL Result:" + str(result))
+                if not result:
+                    # return self._query(f"SHOW COLUMNS FROM {table_name}")
+                    return self.get_simple_fields(table_name)
+                return result
+            else:
+                return self.get_simple_fields(table_name)
+
+    def get_simple_fields(self, table_name):
+        """Get column fields about specified table."""
+        return self._query(f"SHOW COLUMNS FROM {table_name}")
+
+    def get_current_db_name(self):
+        return self.client.database
+
+    def get_table_comments(self, db_name: str):
+        session = self.client
+
+        _query_sql = f"""
+                SELECT table, comment FROM system.tables WHERE database = '{db_name}'""".format(
             db_name
         )
+
+        with session.query_row_block_stream(_query_sql) as stream:
+            table_comments = [row for block in stream for row in block]
+            return table_comments
+
+    def get_table_comment(self, table_name: str) -> Dict:
+        """Get table comment.
+        Args:
+            table_name (str): table name
+        Returns:
+            comment: Dict, which contains text: Optional[str], eg:["text": "comment"]
+        """
+        session = self.client
+
+        _query_sql = f"""
+                SELECT table, comment FROM system.tables WHERE database = '{self.client.database}'and table = '{table_name}'""".format(
+            self.client.database
         )
+
+        with session.query_row_block_stream(_query_sql) as stream:
+            table_comments = [row for block in stream for row in block]
+            return [{"text": comment} for table_name, comment in table_comments][0]
+
+    def get_column_comments(self, db_name, table_name):
+        session = self.client
+        _query_sql = f"""
+            select name column, comment from  system.columns where database='{db_name}' and table='{table_name}'
+        """.format(
+            db_name, table_name
         )
-        table_comments = cursor.fetchall()
+
-        return [
+        with session.query_row_block_stream(_query_sql) as stream:
-            (table_comment[0], table_comment[1]) for table_comment in table_comments
+            column_comments = [row for block in stream for row in block]
-        ]
+            return column_comments
 
     def table_simple_info(self):
         # group_concat() not supported in clickhouse, use arrayStringConcat+groupArray instead; and quotes need to be escaped
-        _sql = f"""
-                select concat(TABLE_NAME, \'(\' , arrayStringConcat(groupArray(column_name),\'-\'), \')\') as schema_info 
-                from information_schema.COLUMNS where table_schema=\'{self.get_current_db_name()}\' group by TABLE_NAME; """
 
-        cursor = self.session.execute(text(_sql))
+        _sql = f"""
-        results = cursor.fetchall()
+            SELECT concat(TABLE_NAME, '(', arrayStringConcat(groupArray(column_name), '-'), ')') AS schema_info
-        return results
+            FROM information_schema.COLUMNS
+            WHERE table_schema = '{self.get_current_db_name()}'
+            GROUP BY TABLE_NAME
+        """
+        with self.client.query_row_block_stream(_sql) as stream:
+            return [row[0] for block in stream for row in block]
+
+    def _write(self, write_sql: str):
+        """write data
+
+        Args:
+            write_sql (str): sql string
+        """
+        # TODO need to be implemented
+        print(f"Write[{write_sql}]")
+        result = self.client.command(write_sql)
+        print(f"SQL[{write_sql}], result:{result.written_rows}")
+
+    def _query(self, query: str, fetch: str = "all"):
+        """Query data from clickhouse
+
+        Args:
+            query (str): sql string
+            fetch (str, optional): "one" or "all". Defaults to "all".
+
+        Raises:
+            ValueError: Error
+
+        Returns:
+            _type_: List<Result>
+        """
+        # TODO need to be implemented
+        print(f"Query[{query}]")
+
+        if not query:
+            return []
+
+        cursor = self.client.query(query)
+        if fetch == "all":
+            result = cursor.result_rows
+        elif fetch == "one":
+            result = cursor.first_row
+        else:
+            raise ValueError("Fetch parameter must be either 'one' or 'all'")
+
+        field_names = cursor.column_names
+        result.insert(0, field_names)
+        return result
+
+    def __sql_parse(self, sql):
+        sql = sql.strip()
+        parsed = sqlparse.parse(sql)[0]
+        sql_type = parsed.get_type()
+        if sql_type == "CREATE":
+            table_name = self._extract_table_name_from_ddl(parsed)
+        else:
+            table_name = parsed.get_name()
+
+        first_token = parsed.token_first(skip_ws=True, skip_cm=False)
+        ttype = first_token.ttype
+        print(f"SQL:{sql}, ttype:{ttype}, sql_type:{sql_type}, table:{table_name}")
+        return parsed, ttype, sql_type, table_name
+
+    def _sync_tables_from_db(self) -> Iterable[str]:
+        """Read table information from database"""
+        # TODO Use a background thread to refresh periodically
+
+        # SQL will raise error with schema
+        _schema = (
+            None if self.db_type == DBType.SQLite.value() else self._engine.url.database
+        )
+        # including view support by adding the views as well as tables to the all
+        # tables list if view_support is True
+        self._all_tables = set(
+            self._inspector.get_table_names(schema=_schema)
+            + (
+                self._inspector.get_view_names(schema=_schema)
+                if self.view_support
+                else []
+            )
+        )
+        return self._all_tables

dbgpt/rag/summary/rdbms_db_summary.py

@@ -76,7 +76,7 @@ def _parse_table_summary(
         table_name(column1(column1 comment),column2(column2 comment),column3(column3 comment) and index keys, and table comment: {table_comment})
     """
     columns = []
-    for column in conn._inspector.get_columns(table_name):
+    for column in conn.get_columns(table_name):
         if column.get("comment"):
             columns.append(f"{column['name']} ({column.get('comment')})")
         else:
@@ -84,7 +84,7 @@ def _parse_table_summary(
 
     column_str = ", ".join(columns)
     index_keys = []
-    for index_key in conn._inspector.get_indexes(table_name):
+    for index_key in conn.get_indexes(table_name):
         key_str = ", ".join(index_key["column_names"])
         index_keys.append(f"{index_key['name']}(`{key_str}`) ")
     table_str = summary_template.format(table_name=table_name, columns=column_str)
@@ -92,7 +92,7 @@ def _parse_table_summary(
         index_key_str = ", ".join(index_keys)
         table_str += f", and index keys: {index_key_str}"
     try:
-        comment = conn._inspector.get_table_comment(table_name)
+        comment = conn.get_table_comment(table_name)
     except Exception:
         comment = dict(text=None)
     if comment.get("text"):

setup.py

@@ -536,6 +536,7 @@ def all_datasource_requires():
         # mysqlclient 2.2.x have pkg-config issue on 3.10+
         "mysqlclient==2.1.0",
         "pydoris>=1.0.2,<2.0.0",
+        "clickhouse-connect",
     ]
 
 

tests/intetration_tests/datasource/test_conn_clickhouse.py

@@ -1,3 +1,23 @@
+"""_Create_table
+        CREATE TABLE default.my_first_table
+        (
+            `user_id` UInt32,
+            `message` String,
+            `timestamp` DateTime,
+            `metric` Float32
+        )
+        ENGINE = MergeTree
+        PRIMARY KEY (user_id, timestamp)
+        ORDER BY (user_id, timestamp);
+        
+    # INSERT INTO my_first_table (user_id, message, timestamp, metric) VALUES
+    (101, 'Hello, ClickHouse!',                                 now(),       -1.0    ),
+    (102, 'Insert a lot of rows per batch',                     yesterday(), 1.41421 ),
+    (102, 'Sort your data based on your commonly-used queries', today(),     2.718   ),
+    (101, 'Granules are the smallest chunks of data read',      now() + 5,   3.14159 )
+    """
+from typing import Dict, List
+
 import pytest
 
 from dbgpt.datasource.rdbms.conn_clickhouse import ClickhouseConnect
@@ -7,3 +27,42 @@ from dbgpt.datasource.rdbms.conn_clickhouse import ClickhouseConnect
 def db():
     conn = ClickhouseConnect.from_uri_db("localhost", 8123, "default", "", "default")
     yield conn
+
+
+def test_create_table(db):
+    _create_sql = """
+        CREATE TABLE IF NOT EXISTS my_first_table
+        (
+            `user_id` UInt32,
+            `message` String,
+            `timestamp` DateTime,
+            `metric` Float32
+        )
+        ENGINE = MergeTree
+        PRIMARY KEY (user_id, timestamp)
+        ORDER BY (user_id, timestamp);
+    """
+    db.run(_create_sql)
+    assert list(db.get_table_names()) == ["my_first_table"]
+
+
+def test_get_table_names(db):
+    assert list(db.get_table_names()) == ["my_first_table"]
+
+
+def test_get_indexes(db):
+    assert [index.get("name") for index in db.get_indexes("my_first_table")][
+        0
+    ] == "primary_key"
+
+
+def test_get_fields(db):
+    assert list(db.get_fields("my_first_table")[0])[0][0] == "user_id"
+
+
+def test_get_table_comments(db):
+    assert db.get_table_comments("my_first_table") == []
+
+
+def test_get_columns_comments(db):
+    assert db.get_column_comments("default", "my_first_table")[0][1] == ""

tests/intetration_tests/datasource/test_conn_mysql.py

@@ -34,7 +34,7 @@ def db():
         "localhost",
         3307,
         "root",
-        "******",
+        "********",
         "test",
         engine_args={"connect_args": {"charset": "utf8mb4"}},
     )

web/README.md

@@ -63,32 +63,16 @@ npm run dev
 npm run compile
 
 # copy compile file to DB-GPT static file dictory
-cp -r -f /Db-GPT-Web/out/* /DB-GPT/pilot/server/static/
+cp -rf out/* ../dbgpt/app/static 
 
 ```
 
 ## 📚 Documentation
 
-For full documentation, visit [document](https://db-gpt.readthedocs.io/en/latest/index.html).
+For full documentation, visit [document](https://docs.dbgpt.site/).
 
-## 📺 Screenshots
-Run on an RTX 4090 GPU.
-
-#### Agent
-![Agent](./screenshots/agent.gif)
-#### ChatExcel
-![ChatExcel](./screenshots/chatexcel.gif)
-#### Knowledge
-![Knowledge](./screenshots/knowledge.gif)
-#### Models
-![models](./screenshots/models.gif)
-#### FastChat
-![FastChat](./screenshots/fastchat.gif)
-#### vllm
-![vllm](./screenshots/vllm.gif)
 
 ## Usage
-
   [react-markdown](https://github.com/remarkjs/react-markdown#readme) for markdown support.
   [ant-design](https://github.com/ant-design/ant-design) for ui components.
   [next.js](https://github.com/vercel/next.js) for server side rendering.