From 909bb263a4b389bda5c96ac41903363074414825 Mon Sep 17 00:00:00 2001 From: FangYin Cheng Date: Fri, 18 Aug 2023 15:12:37 +0800 Subject: [PATCH 1/2] feat: Add examples and modify deploy documents --- docs/getting_started/faq/deploy/deploy_faq.md | 23 +++++-- docs/getting_started/install/deploy/deploy.md | 17 +++-- .../install/llm/llama/llama_cpp.md | 6 +- pilot/connections/rdbms/conn_sqlite.py | 4 ++ pilot/model/conversation.py | 14 ++++ scripts/examples/load_examples.bat | 68 ++++++++++++++++++ scripts/examples/load_examples.sh | 69 +++++++++++++++++++ scripts/llama_cpp_install.sh | 47 +++++++++++++ 8 files changed, 235 insertions(+), 13 deletions(-) create mode 100644 scripts/examples/load_examples.bat create mode 100755 scripts/examples/load_examples.sh create mode 100755 scripts/llama_cpp_install.sh diff --git a/docs/getting_started/faq/deploy/deploy_faq.md b/docs/getting_started/faq/deploy/deploy_faq.md index 0c3091042..42a0e6afa 100644 --- a/docs/getting_started/faq/deploy/deploy_faq.md +++ b/docs/getting_started/faq/deploy/deploy_faq.md @@ -2,26 +2,26 @@ Installation FAQ ================================== -##### Q1: execute `pip install -r requirements.txt` error, found some package cannot find correct version. +##### Q1: execute `pip install -e .` error, found some package cannot find correct version. change the pip source. ```bash # pypi -$ pip install -r requirements.txt -i https://pypi.python.org/simple +$ pip install -e . -i https://pypi.python.org/simple ``` or ```bash # tsinghua -$ pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ +$ pip install -e . -i https://pypi.tuna.tsinghua.edu.cn/simple/ ``` or ```bash # aliyun -$ pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ +$ pip install -e . -i http://mirrors.aliyun.com/pypi/simple/ ``` ##### Q2: sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file @@ -29,5 +29,20 @@ $ pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ make sure you pull latest code or create directory with mkdir pilot/data ##### Q3: The model keeps getting killed. + your GPU VRAM size is not enough, try replace your hardware or replace other llms. +##### Q4: How to access website on the public network + +You can try to use gradio's [network](https://github.com/gradio-app/gradio/blob/main/gradio/networking.py) to achieve. +```python +import secrets +from gradio import networking +token=secrets.token_urlsafe(32) +local_port=5000 +url = networking.setup_tunnel('0.0.0.0', local_port, token) +print(f'Public url: {url}') +time.sleep(60 * 60 * 24) +``` + +Open `url` with your browser to see the website. \ No newline at end of file diff --git a/docs/getting_started/install/deploy/deploy.md b/docs/getting_started/install/deploy/deploy.md index a2595cdff..0422a5c58 100644 --- a/docs/getting_started/install/deploy/deploy.md +++ b/docs/getting_started/install/deploy/deploy.md @@ -49,7 +49,7 @@ For the entire installation process of DB-GPT, we use the miniconda3 virtual env python>=3.10 conda create -n dbgpt_env python=3.10 conda activate dbgpt_env -pip install -r requirements.txt +pip install -e . ``` Before use DB-GPT Knowledge ```bash @@ -97,15 +97,20 @@ You can configure basic parameters in the .env file, for example setting LLM_MOD ### 3. Run +**(Optional) load examples into SQLlite** +```bash +bash ./scripts/examples/load_examples.sh +``` + 1.Run db-gpt server ```bash -$ python pilot/server/dbgpt_server.py +python pilot/server/dbgpt_server.py ``` Open http://localhost:5000 with your browser to see the product. -```tip +```{tip} If you want to access an external LLM service, you need to 1.set the variables LLM_MODEL=YOUR_MODEL_NAME, MODEL_SERVER=YOUR_MODEL_SERVER（eg:http://localhost:5000） in the .env file. @@ -116,7 +121,7 @@ If you want to access an external LLM service, you need to If you want to learn about dbgpt-webui, read https://github./csunny/DB-GPT/tree/new-page-framework/datacenter ```bash -$ python pilot/server/dbgpt_server.py --light +python pilot/server/dbgpt_server.py --light ``` ### Multiple GPUs @@ -141,6 +146,4 @@ DB-GPT supported 8-bit quantization and 4-bit quantization. You can modify the setting `QUANTIZE_8bit=True` or `QUANTIZE_4bit=True` in `.env` file to use quantization(8-bit quantization is enabled by default). -Llama-2-70b with 8-bit quantization can run with 80 GB of VRAM, and 4-bit quantization can run with 48 GB of VRAM. - -Note: you need to install the latest dependencies according to [requirements.txt](https://github.com/eosphoros-ai/DB-GPT/blob/main/requirements.txt). +Llama-2-70b with 8-bit quantization can run with 80 GB of VRAM, and 4-bit quantization can run with 48 GB of VRAM. \ No newline at end of file diff --git a/docs/getting_started/install/llm/llama/llama_cpp.md b/docs/getting_started/install/llm/llama/llama_cpp.md index 7def97d2a..ebb3b8220 100644 --- a/docs/getting_started/install/llm/llama/llama_cpp.md +++ b/docs/getting_started/install/llm/llama/llama_cpp.md @@ -1,6 +1,8 @@ -### llama.cpp +llama.cpp +================================== -DB-GPT is now supported by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) through [llama.cpp](https://github.com/ggerganov/llama.cpp). + +DB-GPT already supports [llama.cpp](https://github.com/ggerganov/llama.cpp) via [llama-cpp-python](https://github.com/abetlen/llama-cpp-python). ## Running llama.cpp diff --git a/pilot/connections/rdbms/conn_sqlite.py b/pilot/connections/rdbms/conn_sqlite.py index 1740537cf..3b980b555 100644 --- a/pilot/connections/rdbms/conn_sqlite.py +++ b/pilot/connections/rdbms/conn_sqlite.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os from typing import Optional, Any, Iterable from sqlalchemy import create_engine, text @@ -24,6 +25,9 @@ class SQLiteConnect(RDBMSDatabase): _engine_args = engine_args or {} _engine_args["connect_args"] = {"check_same_thread": False} # _engine_args["echo"] = True + directory = os.path.dirname(file_path) + if not os.path.exists(directory): + os.makedirs(directory) return cls(create_engine("sqlite:///" + file_path, **_engine_args), **kwargs) def get_indexes(self, table_name): diff --git a/pilot/model/conversation.py b/pilot/model/conversation.py index 5ee54cfc1..355bdda19 100644 --- a/pilot/model/conversation.py +++ b/pilot/model/conversation.py @@ -335,6 +335,20 @@ register_conv_template( ) ) +# Alpaca default template +register_conv_template( + Conversation( + name="alpaca", + system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.", + roles=("### Instruction", "### Response"), + messages=(), + offset=0, + sep_style=SeparatorStyle.ADD_COLON_TWO, + sep="\n\n", + sep2="", + ) +) + # Baichuan-13B-Chat template register_conv_template( # source: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat/blob/f5f47be2adbbdceb784f334d6fa1ca2c73e65097/modeling_baichuan.py#L507 diff --git a/scripts/examples/load_examples.bat b/scripts/examples/load_examples.bat new file mode 100644 index 000000000..2ecf1c507 --- /dev/null +++ b/scripts/examples/load_examples.bat @@ -0,0 +1,68 @@ +@echo off +setlocal + +:: Get script location and set working directory +for %%i in (%0) do set SCRIPT_LOCATION=%%~dpi +cd %SCRIPT_LOCATION% +cd .. +cd .. +set WORK_DIR=%CD% + +:: Check if sqlite3 is installed +where sqlite3 >nul 2>nul +if %ERRORLEVEL% neq 0 ( + echo sqlite3 not found, please install sqlite3 + exit /b 1 +) + +:: Default file paths +set DEFAULT_DB_FILE=DB-GPT\pilot\data\default_sqlite.db +set DEFAULT_SQL_FILE=DB-GPT\docker\examples\sqls\*_sqlite.sql +set DB_FILE=%WORK_DIR%\pilot\data\default_sqlite.db +set SQL_FILE= + +:argLoop +if "%1"=="" goto argDone +if "%1"=="-d" goto setDBFile +if "%1"=="--db-file" goto setDBFile +if "%1"=="-f" goto setSQLFile +if "%1"=="--sql-file" goto setSQLFile +if "%1"=="-h" goto printUsage +if "%1"=="--help" goto printUsage +goto argError + +:setDBFile +shift +set DB_FILE=%1 +shift +goto argLoop + +:setSQLFile +shift +set SQL_FILE=%1 +shift +goto argLoop + +:argError +echo Invalid argument: %1 +goto printUsage + +:printUsage +echo USAGE: %0 [--db-file sqlite db file] [--sql-file sql file path to run] +echo [-d|--db-file sqlite db file path] default: %DEFAULT_DB_FILE% +echo [-f|--sql-file sqlite file to run] default: %DEFAULT_SQL_FILE% +echo [-h|--help] Usage message +exit /b 0 + +:argDone + +if "%SQL_FILE%"=="" ( + if not exist "%WORK_DIR%\pilot\data" mkdir "%WORK_DIR%\pilot\data" + for %%f in (%WORK_DIR%\docker\examples\sqls\*_sqlite.sql) do ( + echo execute sql file: %%f + sqlite3 "%DB_FILE%" < "%%f" + ) +) else ( + echo Execute SQL file %SQL_FILE% + sqlite3 "%DB_FILE%" < "%SQL_FILE%" +) diff --git a/scripts/examples/load_examples.sh b/scripts/examples/load_examples.sh new file mode 100755 index 000000000..0a829bdac --- /dev/null +++ b/scripts/examples/load_examples.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Only support SQLite now + +SCRIPT_LOCATION=$0 +cd "$(dirname "$SCRIPT_LOCATION")" +WORK_DIR=$(pwd) +WORK_DIR="$WORK_DIR/../.." + +if ! command -v sqlite3 > /dev/null 2>&1 +then + echo "sqlite3 not found, please install sqlite3" + exit 1 +fi + +DEFAULT_DB_FILE="DB-GPT/pilot/data/default_sqlite.db" +DEFAULT_SQL_FILE="DB-GPT/docker/examples/sqls/*_sqlite.sql" +DB_FILE="$WORK_DIR/pilot/data/default_sqlite.db" +SQL_FILE="" + +usage () { + echo "USAGE: $0 [--db-file sqlite db file] [--sql-file sql file path to run]" + echo " [-d|--db-file sqlite db file path] default: ${DEFAULT_DB_FILE}" + echo " [-f|--sql-file sqlte file to run] default: ${DEFAULT_SQL_FILE}" + echo " [-h|--help] Usage message" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -d|--db-file) + DB_FILE="$2" + shift # past argument + shift # past value + ;; + -f|--sql-file) + SQL_FILE="$2" + shift + shift + ;; + -h|--help) + help="true" + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if [[ $help ]]; then + usage + exit 0 +fi + +if [ -n $SQL_FILE ];then + mkdir -p $WORK_DIR/pilot/data + for file in $WORK_DIR/docker/examples/sqls/*_sqlite.sql + do + echo "execute sql file: $file" + sqlite3 $DB_FILE < "$file" + done + +else + echo "Execute SQL file ${SQL_FILE}" + sqlite3 $DB_FILE < $SQL_FILE +fi + + diff --git a/scripts/llama_cpp_install.sh b/scripts/llama_cpp_install.sh new file mode 100755 index 000000000..685678ae4 --- /dev/null +++ b/scripts/llama_cpp_install.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Get cuda version by torch +CUDA_VERSION=`python -c "import torch;print(torch.version.cuda.replace('.', '') if torch.cuda.is_available() else '')"` + +DEVICE="cpu" +CPU_OPT="basic" + +if [ "${CUDA_VERSION}" = "" ]; then + echo "CUDA not support, use cpu version" +else + DEVICE="cu${CUDA_VERSION//./}" + echo "CUDA version: $CUDA_VERSION, download path: $DEVICE" +fi + +echo "Checking CPU support:" +CPU_SUPPORT=$(lscpu) + +echo "$CPU_SUPPORT" | grep -q "avx " +if [ $? -eq 0 ]; then + echo "CPU supports AVX." + # CPU_OPT="AVX" + # TODO AVX will failed on my cpu +else + echo "CPU does not support AVX." +fi + +echo "$CPU_SUPPORT" | grep -q "avx2" +if [ $? -eq 0 ]; then + echo "CPU supports AVX2." + CPU_OPT="AVX2" +else + echo "CPU does not support AVX2." +fi + +echo "$CPU_SUPPORT" | grep -q "avx512" +if [ $? -eq 0 ]; then + echo "CPU supports AVX512." + CPU_OPT="AVX512" +else + echo "CPU does not support AVX512." +fi + +EXTRA_INDEX_URL="https://jllllll.github.io/llama-cpp-python-cuBLAS-wheels/$CPU_OPT/$DEVICE" + +echo "install llama-cpp-python from --extra-index-url ${EXTRA_INDEX_URL}" +python -m pip install llama-cpp-python --force-reinstall --no-cache --prefer-binary --extra-index-url=$EXTRA_INDEX_URL From efd17593ce9e947f405b109863f2d3875047e279 Mon Sep 17 00:00:00 2001 From: FangYin Cheng Date: Fri, 18 Aug 2023 17:18:01 +0800 Subject: [PATCH 2/2] fix: Fix alpaca prompt error --- pilot/model/conversation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pilot/model/conversation.py b/pilot/model/conversation.py index 355bdda19..8508881b7 100644 --- a/pilot/model/conversation.py +++ b/pilot/model/conversation.py @@ -339,7 +339,7 @@ register_conv_template( register_conv_template( Conversation( name="alpaca", - system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.", + system="Below is an instruction that describes a task. Write a response that appropriately completes the request.", roles=("### Instruction", "### Response"), messages=(), offset=0,