github/ColossalAI
1
0
Fork 0
mirror of https://github.com/hpcaitech/ColossalAI.git synced 2025-09-07 03:52:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bugs in request_handler.py and engine.py

Browse Source
This commit is contained in:
yuehuayingxueluo
2024-01-10 10:38:53 +08:00
committed by FrankLeeeee
parent 10e3c9f923
commit d40eb26029
4 changed files with 21 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/inference/core/request_handler.py
View File

@@ -131,9 +131,9 @@ class RequestHandler:
"""
assert not self._find_sequence(req.request_id), f"Sequence {req.request_id} already exists."
assert (
req.input_len < self.inference_config.max_input_len
req.input_len <= self.inference_config.max_input_len
), f"Sequence {req.request_id} exceeds input length limit"
self.waiting_list[req.input_len * 3 // self.inference_config.max_input_len].append(req)
self.waiting_list[req.input_len * 3 // (self.inference_config.max_input_len + 1)].append(req)
def abort_sequence(self, request_id: str):
"""