[BugFix][MRV2] Don't create dummy requests longer than max_model_len (#49751)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 02:04:20 +00:00
committed by GitHub
parent f0553889c0
commit 50aa830482
4 changed files with 72 additions and 7 deletions
@@ -0,0 +1,52 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the V2 model runner's InputBatch (vllm.v1.worker.gpu.input_batch)."""
import pytest
import torch
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
DEVICE = "cuda"
@pytest.mark.parametrize(
"num_reqs,num_tokens",
[
(256, 496), # remainder 240: previously gave the last request 241 tokens
(128, 512), # no remainder
(3, 8),
(1, 7),
],
)
def test_make_dummy_distributes_remainder(num_reqs: int, num_tokens: int):
"""No dummy request may exceed ceil(num_tokens / num_reqs) tokens.
Dumping the remainder on a single request can produce a dummy request with
seq_len > max_model_len, which the block tables cannot back; attention
kernels running on the dummy batch during cudagraph capture then read
block-table entries out of bounds (https://github.com/vllm-project/vllm/pull/49364
CI failure).
"""
buffers = InputBuffers(
max_num_reqs=num_reqs, max_num_tokens=num_tokens, device=torch.device(DEVICE)
)
batch = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
max_per_req = -(-num_tokens // num_reqs)
assert batch.num_scheduled_tokens.sum() == num_tokens
assert batch.num_scheduled_tokens.max() == max_per_req
assert batch.num_scheduled_tokens.min() >= num_tokens // num_reqs
# Requests with an extra token are placed at the end of the batch.
assert (batch.num_scheduled_tokens[:-1] <= batch.num_scheduled_tokens[1:]).all()
# seq_len == query_len for the dummy prefill-shaped batch, on GPU and CPU.
query_lens = batch.query_start_loc_np[1:] - batch.query_start_loc_np[:-1]
assert (query_lens == batch.num_scheduled_tokens).all()
assert torch.equal(
batch.seq_lens, torch.from_numpy(batch.num_scheduled_tokens).to(DEVICE)
)
assert batch.query_start_loc_np[-1] == num_tokens
assert torch.equal(
batch.query_start_loc.cpu(), torch.from_numpy(batch.query_start_loc_np)
)