From 4702d5784145b6164fe3dd90b35ba32fcd85b491 Mon Sep 17 00:00:00 2001 From: Tong Li Date: Thu, 13 Mar 2025 16:49:02 +0800 Subject: [PATCH] convert to 8 generation --- .../ColossalChat/coati/distributed/inference_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/ColossalChat/coati/distributed/inference_backend.py b/applications/ColossalChat/coati/distributed/inference_backend.py index bc0ae5c36..8711d0b8c 100644 --- a/applications/ColossalChat/coati/distributed/inference_backend.py +++ b/applications/ColossalChat/coati/distributed/inference_backend.py @@ -154,7 +154,7 @@ class VLLMInferenceBackend(BaseInferenceBackend): ) FORCE_GENERATE_CONFIG = dict( logprobs=0, - n=4, + n=8, ) def __init__(self, model_config: Dict[str, Any], generate_config: Dict[str, Any], tokenizer: PreTrainedTokenizer):