[BugFix] Stop dummy runs from writing mamba state through stale block-table rows (#49757)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Jeff Ma <jeffjma@umich.edu>
This commit is contained in:
@@ -130,3 +130,66 @@ def test_block_tables_apply_staged_writes_single_group():
|
||||
block_tables.block_tables[0].gpu[0, :2],
|
||||
torch.tensor([1, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
|
||||
def test_v1_block_table_move_row_clears_vacated_row():
|
||||
"""condense() moves the last row into a freed slot; the vacated row must
|
||||
not keep stale block ids. Padded dummy-run batches dereference stale rows
|
||||
as mamba state slots (bypassing the NULL_BLOCK_ID fill of real decode
|
||||
padding) and write state in place there — corrupting the blocks' new
|
||||
owner once they are reallocated, e.g. to an in-flight NIXL load."""
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
|
||||
block_table = BlockTable(
|
||||
block_size=16,
|
||||
max_num_reqs=4,
|
||||
max_num_blocks_per_req=8,
|
||||
max_num_batched_tokens=64,
|
||||
pin_memory=False,
|
||||
device=torch.device("cuda"),
|
||||
kernel_block_size=16,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
block_table.add_row([7, 8, 9], row_idx=0)
|
||||
block_table.add_row([4, 5], row_idx=1)
|
||||
|
||||
block_table.move_row(1, 0)
|
||||
|
||||
assert block_table.block_table.np[0, :2].tolist() == [4, 5]
|
||||
assert block_table.num_blocks_per_row[0] == 2
|
||||
# The vacated source row routes to the reserved null block.
|
||||
assert block_table.num_blocks_per_row[1] == 0
|
||||
assert (block_table.block_table.np[1] == 0).all()
|
||||
|
||||
|
||||
def test_get_dummy_block_tables_returns_zeroed_rows():
|
||||
"""Dummy runs bypass the gather, so the persistent input_block_tables
|
||||
hold the previous real step's rows. Mamba/GDN metadata routes in-place
|
||||
state writes through block_table[:, 0] (dummy slot mappings are
|
||||
PAD-filled, state indices are not), so stale rows would direct dummy
|
||||
state writes at freed — possibly reallocated — blocks.
|
||||
get_dummy_block_tables must hand out zeroed (null block) rows while
|
||||
preserving the persistent storage address for CUDA graphs."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[16],
|
||||
max_num_reqs=4,
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
# Simulate a real step: stage a request's blocks and gather them into
|
||||
# the persistent input block tables.
|
||||
block_tables.append_block_ids(req_index=0, new_block_ids=([1, 2],), overwrite=True)
|
||||
block_tables.apply_staged_writes()
|
||||
idx_mapping = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
block_tables.gather_block_tables(idx_mapping, num_reqs_padded=1)
|
||||
torch.accelerator.synchronize()
|
||||
assert block_tables.input_block_tables[0][0, 0].item() == 1
|
||||
|
||||
dummy = block_tables.get_dummy_block_tables(num_reqs=1)
|
||||
torch.accelerator.synchronize()
|
||||
assert (dummy[0] == 0).all()
|
||||
# CUDA graph invariant: same persistent tensor, not a fresh allocation.
|
||||
assert dummy[0].data_ptr() == block_tables.input_block_tables[0].data_ptr()
|
||||
|
||||
Reference in New Issue
Block a user