github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-06 11:32:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bugs in sampler

Browse Source
This commit is contained in:
yuehuayingxueluo
2024-01-04 15:03:18 +08:00
committed by FrankLeeeee
parent 02c1bf8b2a
commit bbfebfb9fc
4 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/inference/core/request_handler.py
View File

@@ -180,9 +180,9 @@ class RequestHandler:
"""
# do logit processor
# NOTE: need to decide the granularity to process logits (sequence or batch)
for type in ["top_p", "top_k", "min_p"]:
for type in ["top_k", "top_p", "min_p"]:
config_dict = generation_config.to_dict()
if type in config_dict:
if type in config_dict and config_dict[type] is not None:
logits = logit_processor(type, logits, config_dict[type])
torch.cuda.synchronize()