github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-05 11:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

[hotfix] fix inference typo (#5438)

Browse Source
This commit is contained in:
hugo-syn
2024-05-13 15:06:44 +02:00
committed by GitHub
parent 785cd9a9c9
commit 393c8f5b7f
2 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
colossalai/legacy/inference/async_manager.py
View File

@@ -55,14 +55,14 @@ class Async_DynamicBatchManager(DynamicBatchManager):
self.stats_tool.count_prompt_tokens(new_batch)
self.running_batch = new_batch
has_new_finished, outputs = self._prefill_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens = 0
else:
if self.has_wait_tokens < self.max_wait_tokens:
self.stats_tool.count_output_tokens(self.running_batch)
has_new_finished, outputs = self._decode_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens += 1
else:
@@ -78,7 +78,7 @@ class Async_DynamicBatchManager(DynamicBatchManager):
else:
self.stats_tool.count_output_tokens(self.running_batch)
has_new_finished, outputs = self._decode_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens += 1
if has_new_finished: