github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-01 17:17:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Online Server] Chat Api for streaming and not streaming response (#5470)

Browse Source 
* fix bugs

* fix bugs

* fix api server

* fix api server

* add chat api and test

* del request.n
This commit is contained in:
Jianghai
2024-04-07 14:45:43 +08:00
committed by CjhHa1
parent de378cd2ab
commit c064032865
7 changed files with 326 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
examples/inference/client/locustfile.py
View File

@@ -14,9 +14,37 @@ class QuickstartUser(HttpUser):
def completion_streaming(self):
self.client.post("/v1/completion", json={"prompt": "hello, who are you? ", "stream": "True"})
@tag("online-chat")
@task(5)
def chat(self):
self.client.post(
"v1/chat",
json={
"converation": [
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "what is 1+1?"},
],
"stream": "False",
},
)
@tag("online-chat")
@task(5)
def chat_streaming(self):
self.client.post(
"v1/chat",
json={
"converation": [
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "what is 1+1?"},
],
"stream": "True",
},
)
@tag("offline-generation")
@task(5)
def generate_stream(self):
def generate_streaming(self):
self.client.post("/generate", json={"prompt": "Can you help me? ", "stream": "True"})
@tag("offline-generation")

7
examples/inference/client/run_locust.sh
View File

@@ -4,9 +4,10 @@
# launch server
model_path=${1:-"lmsys/vicuna-7b-v1.3"}
chat_template="{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}"
echo "Model Path: $model_path"
echo "Starting server..."
python -m colossalai.inference.server.api_server --model $model_path &
python -m colossalai.inference.server.api_server --model $model_path --chat-template $chat_template &
SERVER_PID=$!
# waiting time
@@ -15,8 +16,10 @@ sleep 60
# Run Locust
echo "Starting Locust..."
echo "The test will automatically begin, you can turn to http://0.0.0.0:8089 for more information."
echo "Test completion api first"
locust -f locustfile.py -t 300 --tags online-generation --host http://127.0.0.1:8000 --autostart --users 100 --stop-timeout 10
echo "Test chat api"
locust -f locustfile.py -t 300 --tags online-chat --host http://127.0.0.1:8000 --autostart --users 100 --stop-timeout 10
# kill Server
echo "Stopping server..."
kill $SERVER_PID