github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-08 04:23:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(model): Fix vllm new tokenizer error (#1601)

Browse Source
This commit is contained in:
Fangyin Cheng
2024-06-05 15:27:58 +08:00
committed by GitHub
parent c3c063683c
commit 43b5821ce4
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dbgpt/model/adapter/vllm_adapter.py
View File

@@ -40,7 +40,7 @@ class VLLMModelAdapterWrapper(LLMModelAdapter):
help="local model path of the huggingface model to use",
)
parser.add_argument("--model_type", type=str, help="model type")
parser.add_argument("--device", type=str, default=None, help="device")
# parser.add_argument("--device", type=str, default=None, help="device")
# TODO parse prompt templete from `model_name` and `model_path`
parser.add_argument(
"--prompt_template",
@@ -76,7 +76,11 @@ class VLLMModelAdapterWrapper(LLMModelAdapter):
# Set the attributes from the parsed arguments.
engine_args = AsyncEngineArgs(**vllm_engine_args_dict)
engine = AsyncLLMEngine.from_engine_args(engine_args)
return engine, engine.engine.tokenizer
tokenizer = engine.engine.tokenizer
if hasattr(tokenizer, "tokenizer"):
# vllm >= 0.2.7
tokenizer = tokenizer.tokenizer
return engine, tokenizer
def support_async(self) -> bool:
return True