github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-06-21 21:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

[hotfix] fix inference typo (#5438)

Browse Source
This commit is contained in:
hugo-syn 2024-05-13 15:06:44 +02:00 committed by GitHub
parent 785cd9a9c9
commit 393c8f5b7f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
colossalai/legacy/inference/async_manager.py
View File

@ -55,14 +55,14 @@ class Async_DynamicBatchManager(DynamicBatchManager):
self.stats_tool.count_prompt_tokens(new_batch) self.stats_tool.count_prompt_tokens(new_batch)
self.running_batch = new_batch self.running_batch = new_batch
has_new_finished, outputs = self._prefill_batch(self.running_batch) has_new_finished, outputs = self._prefill_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens = 0 self.has_wait_tokens = 0
else: else:
if self.has_wait_tokens < self.max_wait_tokens: if self.has_wait_tokens < self.max_wait_tokens:
self.stats_tool.count_output_tokens(self.running_batch) self.stats_tool.count_output_tokens(self.running_batch)
has_new_finished, outputs = self._decode_batch(self.running_batch) has_new_finished, outputs = self._decode_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens += 1 self.has_wait_tokens += 1
else: else:
@ -78,7 +78,7 @@ class Async_DynamicBatchManager(DynamicBatchManager):
else: else:
self.stats_tool.count_output_tokens(self.running_batch) self.stats_tool.count_output_tokens(self.running_batch)
has_new_finished, outputs = self._decode_batch(self.running_batch) has_new_finished, outputs = self._decode_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens += 1 self.has_wait_tokens += 1
if has_new_finished: if has_new_finished:

8
colossalai/legacy/inference/manager.py
View File

@ -131,14 +131,14 @@ class DynamicBatchManager:
self.stats_tool.count_prompt_tokens(new_batch) self.stats_tool.count_prompt_tokens(new_batch)
self.running_batch = new_batch self.running_batch = new_batch
yield from self._prefill_batch(self.running_batch) yield from self._prefill_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens = 0 self.has_wait_tokens = 0
return return
if self.has_wait_tokens < self.max_wait_tokens: if self.has_wait_tokens < self.max_wait_tokens:
self.stats_tool.count_output_tokens(self.running_batch) self.stats_tool.count_output_tokens(self.running_batch)
yield from self._decode_batch(self.running_batch) yield from self._decode_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens += 1 self.has_wait_tokens += 1
return return
else: else:
@ -154,7 +154,7 @@ class DynamicBatchManager:
else: else:
self.stats_tool.count_output_tokens(self.running_batch) self.stats_tool.count_output_tokens(self.running_batch)
yield from self._decode_batch(self.running_batch) yield from self._decode_batch(self.running_batch)
self._filter_runing_batch() self._filter_running_batch()
self.has_wait_tokens += 1 self.has_wait_tokens += 1
return return
@ -243,7 +243,7 @@ class DynamicBatchManager:
self._filter_batch(batch) self._filter_batch(batch)
yield from self._output_process(finished_reqs) yield from self._output_process(finished_reqs)
def _filter_runing_batch(self): def _filter_running_batch(self):
if self.running_batch is not None and self.running_batch.is_clear(): if self.running_batch is not None and self.running_batch.is_clear():
self.running_batch = None self.running_batch = None