github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-05 19:13:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

[hotfix] fix inference typo (#5438)

Browse Source
This commit is contained in:
hugo-syn
2024-05-13 15:06:44 +02:00
committed by GitHub
parent 785cd9a9c9
commit 393c8f5b7f
2 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
colossalai/legacy/inference/manager.py
View File

@@ -131,14 +131,14 @@ class DynamicBatchManager:
self.stats_tool.count_prompt_tokens(new_batch)
self.running_batch = new_batch
yield from self._prefill_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens = 0
return
if self.has_wait_tokens < self.max_wait_tokens:
self.stats_tool.count_output_tokens(self.running_batch)
yield from self._decode_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens += 1
return
else:
@@ -154,7 +154,7 @@ class DynamicBatchManager:
else:
self.stats_tool.count_output_tokens(self.running_batch)
yield from self._decode_batch(self.running_batch)
self._filter_runing_batch()
self._filter_running_batch()
self.has_wait_tokens += 1
return
@@ -243,7 +243,7 @@ class DynamicBatchManager:
self._filter_batch(batch)
yield from self._output_process(finished_reqs)
def _filter_runing_batch(self):
def _filter_running_batch(self):
if self.running_batch is not None and self.running_batch.is_clear():
self.running_batch = None