[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 02:04:20 +00:00
committed by GitHub
parent f0553889c0
commit 50aa830482
4 changed files with 72 additions and 7 deletions
+5 -1
View File
@@ -59,8 +59,12 @@ def create_lora_capture_hook(
return None
def hook(num_active_loras: int, num_reqs: int, num_tokens: int) -> None:
# Match InputBatch.make_dummy: distribute the remainder evenly so no
# dummy request exceeds ceil(num_tokens / num_reqs) tokens.
num_scheduled = np.full(num_reqs, num_tokens // num_reqs, dtype=np.int32)
num_scheduled[-1] += num_tokens % num_reqs
num_extra = num_tokens % num_reqs
if num_extra > 0:
num_scheduled[-num_extra:] += 1
with runner.maybe_select_dummy_loras(
lora_config, num_scheduled, num_active_loras=num_active_loras
):