[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 02:04:20 +00:00
committed by GitHub
parent f0553889c0
commit 50aa830482
4 changed files with 72 additions and 7 deletions
+6 -2
View File
@@ -564,8 +564,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_tokens = max(num_tokens, self.decode_query_len)
num_reqs = num_tokens // self.decode_query_len
assert num_tokens % self.decode_query_len == 0
num_tokens_per_request = [num_tokens // num_reqs] * num_reqs
num_tokens_per_request[-1] += num_tokens % num_reqs
# Distribute the remainder evenly so no dummy request exceeds
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
num_tokens_per_request = [
num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs)
for i in range(num_reqs)
]
assert sum(num_tokens_per_request) == num_tokens
num_scheduled_tokens = {