mirror of
https://github.com/csunny/DB-GPT.git
synced 2025-10-23 10:20:01 +00:00
refactor: The first refactored version for sdk release (#907)
Co-authored-by: chengfangyin2 <chengfangyin3@jd.com>
This commit is contained in:
8
dbgpt/model/llm_out/llama_cpp_llm.py
Normal file
8
dbgpt/model/llm_out/llama_cpp_llm.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from typing import Dict
|
||||
import torch
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate_stream(model, tokenizer, params: Dict, device: str, context_len: int):
|
||||
# Just support LlamaCppModel
|
||||
return model.generate_streaming(params=params, context_len=context_len)
|
Reference in New Issue
Block a user