[Bugfix] Index Mamba align state copies by request slot in MRV2 (#60210)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Karl0007 <44745209+Karl0007@users.noreply.github.com> Co-authored-by: Yongqin Wang <yongqinwang@roblox.com>
This commit is contained in:
co-authored by
Karl0007
Yongqin Wang
parent
956960f85f
commit
8a26869bca
@@ -22,6 +22,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
MambaCopyBuffers,
|
||||
@@ -638,6 +639,51 @@ def test_mamba_groups_support_mixed_specs_in_uniform_group():
|
||||
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA")
|
||||
def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
|
||||
"""V2 state copies may run steps after their batch (PP), so they must read
|
||||
the per-request-slot tables; aligned state indices are computed for the
|
||||
current batch and keep reading the gathered tables."""
|
||||
cfg = _TestConfig(num_layers=1)
|
||||
device = torch.device("cuda")
|
||||
kv_cache_config = _make_kv_cache_config(cfg, ["layer_0"])
|
||||
forward_context = {
|
||||
"layer_0": _make_mock_attention(
|
||||
torch.empty(cfg.num_blocks, cfg.conv_width, cfg.conv_inner_dim),
|
||||
torch.empty(cfg.num_blocks, cfg.temporal_state_dim),
|
||||
)
|
||||
}
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[cfg.block_size],
|
||||
max_num_reqs=cfg.max_num_reqs,
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[cfg.block_size],
|
||||
)
|
||||
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state._align_mode = True
|
||||
state._mamba_group_ids = []
|
||||
state._mamba_spec = None
|
||||
state._mamba_state_copy_funcs = _COPY_FUNCS
|
||||
state.max_num_reqs = cfg.max_num_reqs
|
||||
state.device = device
|
||||
state.vllm_config = MagicMock()
|
||||
state.vllm_config.compilation_config.static_forward_context = forward_context
|
||||
|
||||
state.initialize_kv_cache(kv_cache_config, block_tables)
|
||||
|
||||
ctx = state._mamba_ctx
|
||||
assert ctx is not None and ctx.is_initialized
|
||||
assert ctx.block_table_ptrs.tolist() == [
|
||||
block_tables.block_tables[0].gpu.data_ptr()
|
||||
]
|
||||
assert ctx.aligned_index_block_table_ptrs.tolist() == [
|
||||
block_tables.input_block_tables[0].data_ptr()
|
||||
]
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user