mirror of
https://github.com/csunny/DB-GPT.git
synced 2025-07-27 05:47:47 +00:00
10 lines
257 B
Python
10 lines
257 B
Python
from typing import Dict
|
|
|
|
import torch
|
|
|
|
|
|
@torch.inference_mode()
|
|
def generate_stream(model, tokenizer, params: Dict, device: str, context_len: int):
|
|
# Just support LlamaCppModel
|
|
return model.generate_streaming(params=params, context_len=context_len)
|