refactor: The first refactored version for sdk release (#907)

Co-authored-by: chengfangyin2 <chengfangyin3@jd.com>
This commit is contained in:
FangYin Cheng
2023-12-08 14:45:59 +08:00
committed by GitHub
parent e7e4aff667
commit cd725db1fb
573 changed files with 2094 additions and 3571 deletions

View File

@@ -0,0 +1,8 @@
from typing import Dict
import torch
@torch.inference_mode()
def generate_stream(model, tokenizer, params: Dict, device: str, context_len: int):
# Just support LlamaCppModel
return model.generate_streaming(params=params, context_len=context_len)