[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 02:04:20 +00:00
committed by GitHub
parent f0553889c0
commit 50aa830482
4 changed files with 72 additions and 7 deletions
+9 -4
View File
@@ -115,13 +115,18 @@ class InputBatch:
expanded_idx_mapping = idx_mapping
expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device)
num_scheduled_tokens = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
num_scheduled_tokens[-1] += num_tokens % num_reqs
# Distribute the remainder evenly so that no dummy request exceeds
# ceil(num_tokens / num_reqs) <= max_model_len tokens.
base_tokens = num_tokens // num_reqs
num_extra = num_tokens % num_reqs
num_scheduled_tokens = np.full(num_reqs, base_tokens, dtype=np.int32)
if num_extra > 0:
num_scheduled_tokens[-num_extra:] += 1
assert int(num_scheduled_tokens.sum()) == num_tokens
# seq_len equals to query_len
input_buffers.seq_lens[:num_reqs] = num_tokens // num_reqs
input_buffers.seq_lens[num_reqs - 1] += num_tokens % num_reqs
input_buffers.seq_lens[: num_reqs - num_extra] = base_tokens
input_buffers.seq_lens[num_reqs - num_extra : num_reqs] = base_tokens + 1
# Pad for full CUDA graph mode.
input_buffers.seq_lens[num_reqs:] = 0
seq_lens = input_buffers.seq_lens[:num_reqs]