[Bugfix] Index Mamba align state copies by request slot in MRV2 (#60210)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Karl0007 <44745209+Karl0007@users.noreply.github.com>
Co-authored-by: Yongqin Wang <yongqinwang@roblox.com>
This commit is contained in:
Jiangyun Zhu
2026-10-07 02:32:46 +00:00
committed by GitHub
co-authored by Karl0007 Yongqin Wang
parent 956960f85f
commit 8a26869bca
6 changed files with 195 additions and 57 deletions
@@ -31,7 +31,11 @@ import torch
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
from vllm.platforms import current_platform
from vllm.v1.worker import mamba_utils as worker_mamba_utils
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, precopy_mamba_align_fused_kernel
from vllm.v1.worker.mamba_utils import (
_TEMPORAL_TILES,
postprocess_mamba_fused_kernel,
precopy_mamba_align_fused_kernel,
)
_parametrize: Callable[..., Callable[[Any], Any]]
@@ -195,7 +199,8 @@ def test_precopy_matches_v1_copy_specs(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
# Non-identity mapping: block-table rows are request slots, not batch rows.
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device).flip(0)
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
precopy_mamba_align_fused_kernel[grid](
dst_col,
@@ -225,6 +230,77 @@ def test_precopy_matches_v1_copy_specs(
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
@_parametrize("conv_state_dim_first", [False, True])
@_cuda_required
def test_v2_postprocess_indexes_block_tables_by_req_slot(conv_state_dim_first):
"""Under PP the V2 postprocess runs after later steps have re-gathered the
batch-ordered tables, so it must index the request-slot tables by slot."""
device = torch.device("cuda")
torch.manual_seed(0)
num_reqs = 4
block_size = 8
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.arange(1, num_blocks, dtype=torch.int32, device=device).view(
num_reqs, MAX_COLS
)
# slot 0: copy col 1 -> 0; slot 1: src == dst; slot 2: not aligned;
# slot 3: copy col 2 -> 0 with one accepted draft token.
num_accepted = torch.tensor([1, 1, 1, 2], dtype=torch.int32, device=device)
state_idx = torch.tensor([1, 0, 1, 2], dtype=torch.int32, device=device)
new_num_computed = torch.tensor([8, 8, 5, 8], dtype=torch.int32, device=device)
ref_src = torch.tensor([1, 0, -1, 2], dtype=torch.int32)
ref_dst = torch.zeros(num_reqs, dtype=torch.int32)
ref_bias = torch.tensor([0, 0, 0, 1], dtype=torch.int32)
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
conv_ref, ssm_ref = _reference(
convs,
ssms,
bt.cpu(),
ref_src,
ref_dst,
ref_bias,
num_reqs,
conv_state_dim_first,
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device=device)
postprocess_mamba_fused_kernel[(num_reqs, NUM_LAYERS * 2, 1)](
num_accepted.clone(),
state_idx,
None,
new_num_computed,
None,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
num_accepted,
idx_mapping,
num_reqs,
block_size=block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=conv_state_dim_first,
HAS_IDX_MAPPING=True,
PRECOMPUTED_NEW_COMPUTED=True,
TEMPORAL_TILES=1,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
def test_ds_conv_copy_spec_reproduces_multi_accept_assert(monkeypatch):
monkeypatch.setattr(
layer_mamba_utils,
+46
View File
@@ -22,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.mamba_utils import (
MambaCopyBuffers,
@@ -638,6 +639,51 @@ def test_mamba_groups_support_mixed_specs_in_uniform_group():
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA")
def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
"""V2 state copies may run steps after their batch (PP), so they must read
the per-request-slot tables; aligned state indices are computed for the
current batch and keep reading the gathered tables."""
cfg = _TestConfig(num_layers=1)
device = torch.device("cuda")
kv_cache_config = _make_kv_cache_config(cfg, ["layer_0"])
forward_context = {
"layer_0": _make_mock_attention(
torch.empty(cfg.num_blocks, cfg.conv_width, cfg.conv_inner_dim),
torch.empty(cfg.num_blocks, cfg.temporal_state_dim),
)
}
block_tables = BlockTables(
block_sizes=[cfg.block_size],
max_num_reqs=cfg.max_num_reqs,
max_num_batched_tokens=64,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[cfg.block_size],
)
state = object.__new__(MambaHybridModelState)
state._align_mode = True
state._mamba_group_ids = []
state._mamba_spec = None
state._mamba_state_copy_funcs = _COPY_FUNCS
state.max_num_reqs = cfg.max_num_reqs
state.device = device
state.vllm_config = MagicMock()
state.vllm_config.compilation_config.static_forward_context = forward_context
state.initialize_kv_cache(kv_cache_config, block_tables)
ctx = state._mamba_ctx
assert ctx is not None and ctx.is_initialized
assert ctx.block_table_ptrs.tolist() == [
block_tables.block_tables[0].gpu.data_ptr()
]
assert ctx.aligned_index_block_table_ptrs.tolist() == [
block_tables.input_block_tables[0].data_ptr()
]
# -----------------------------------------------------------------------------
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
# -----------------------------------------------------------------------------