github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-04 01:50:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

rebuild params

Browse Source
This commit is contained in:
csunny 2023-04-30 21:54:49 +08:00
parent f0e17ed8f1
commit eef244fe92
1 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pilot/server/vicuna_server.py
View File

@ -33,6 +33,13 @@ class PromptRequest(BaseModel):
stop: Optional[List[str]] = None stop: Optional[List[str]] = None
class StreamRequest(BaseModel):
model: str
prompt: str
temperature: float
max_new_tokens: int
stop: str
class EmbeddingRequest(BaseModel): class EmbeddingRequest(BaseModel):
prompt: str prompt: str
@ -64,10 +71,16 @@ def generate_stream_gate(params):
@app.post("/generate_stream") @app.post("/generate_stream")
def api_generate_stream(request: Request): def api_generate_stream(request: StreamRequest):
global model_semaphore, global_counter global model_semaphore, global_counter
global_counter += 1 global_counter += 1
params = request.json() params = {
"prompt": request.prompt,
"model": request.model,
"temperature": request.temperature,
"max_new_tokens": request.max_new_tokens,
"stop": request.stop
}
print(model, tokenizer, params, DEVICE) print(model, tokenizer, params, DEVICE)
# if model_semaphore is None: # if model_semaphore is None:
# model_semaphore = asyncio.Semaphore(LIMIT_MODEL_CONCURRENCY) # model_semaphore = asyncio.Semaphore(LIMIT_MODEL_CONCURRENCY)