[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -564,8 +564,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_tokens = max(num_tokens, self.decode_query_len)
|
||||
num_reqs = num_tokens // self.decode_query_len
|
||||
assert num_tokens % self.decode_query_len == 0
|
||||
num_tokens_per_request = [num_tokens // num_reqs] * num_reqs
|
||||
num_tokens_per_request[-1] += num_tokens % num_reqs
|
||||
# Distribute the remainder evenly so no dummy request exceeds
|
||||
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
|
||||
num_tokens_per_request = [
|
||||
num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs)
|
||||
for i in range(num_reqs)
|
||||
]
|
||||
|
||||
assert sum(num_tokens_per_request) == num_tokens
|
||||
num_scheduled_tokens = {
|
||||
|
||||
Reference in New Issue
Block a user