github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-17 23:18:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

support multi process to launch llmserver, add openai proxy api.

Browse Source
This commit is contained in:
xuyuan23
2023-06-12 21:29:16 +08:00
parent 62eb6c383e
commit 6e3b48c7c4
4 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pilot/server/llmserver.py
View File

@@ -84,6 +84,11 @@ class ModelWorker:
return get_embeddings(self.model, self.tokenizer, prompt)
model_path = LLM_MODEL_CONFIG[CFG.LLM_MODEL]
worker = ModelWorker(
model_path=model_path, model_name=CFG.LLM_MODEL, device=DEVICE, num_gpus=1
)
app = FastAPI()
@@ -157,11 +162,4 @@ def embeddings(prompt_request: EmbeddingRequest):
if __name__ == "__main__":
model_path = LLM_MODEL_CONFIG[CFG.LLM_MODEL]
print(model_path, DEVICE)
worker = ModelWorker(
model_path=model_path, model_name=CFG.LLM_MODEL, device=DEVICE, num_gpus=1
)
uvicorn.run(app, host="0.0.0.0", port=CFG.MODEL_PORT, log_level="info")