github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-22 18:09:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bugs in request_handler

Browse Source
This commit is contained in:
yuehuayingxueluo
2024-01-02 13:02:20 +08:00
committed by FrankLeeeee
parent 62fd08ee44
commit 62968588d1
5 changed files with 21 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
colossalai/inference/core/engine.py
View File

@@ -226,12 +226,15 @@ class InferenceEngine:
self.v_cache,
)
logits = logits[:, -1, :]
self.request_handler.search_tokens(self.generation_config, logits)
finished_sequences = self.request_handler.update()
print("finished_sequences: ", finished_sequences)
# Decode completed sentences.
for seq in finished_sequences:
print("seq.output_token_id: ", seq.output_token_id)
if seq.prompt:
output_str = self.tokenizer.decode(seq.output_token_id, skip_special_tokens=True)
output_list.append(seq.prompt + output_str)
@@ -239,4 +242,6 @@ class InferenceEngine:
output_str = self.tokenizer.decode(seq.input_token_id + seq.output_token_id, skip_special_tokens=True)
output_list.append(output_str)
print("len(output_list): ", len(output_list))
return output_list