[Bugfix][Mamba] Fix overlapping state copy race (#50729)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-17 08:29:41 +02:00
committed by GitHub
co-authored by OpenAI Codex
parent 93550cc4cd
commit a02cfccbc6
2 changed files with 228 additions and 63 deletions
+152 -37
View File
@@ -18,6 +18,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpe
from vllm.v1.worker.mamba_utils import (
MambaCopyBuffers,
MambaSpecDecodeGPUContext,
batch_memcpy,
collect_mamba_copy_meta,
do_mamba_copy_block,
preprocess_mamba,
@@ -536,6 +537,34 @@ class TestPostprocessMambaFusedKernel:
def test_config(self):
return _TestConfig()
def test_batch_memcpy_left_overlap_has_memmove_semantics(self, device):
batch = 128
row_bytes = 32 * 1024
shift = 16
copy_size = row_bytes - shift
pattern = (torch.arange(row_bytes, dtype=torch.int32, device=device) % 251).to(
torch.uint8
)
state = pattern.expand(batch, -1).clone()
snapshot = state.clone()
row_stride_bytes = state.stride(0) * state.element_size()
row_offsets = (
torch.arange(batch, dtype=torch.int64, device=device) * row_stride_bytes
)
dst_ptrs = (row_offsets + state.data_ptr()).to(torch.uint64)
src_ptrs = (row_offsets + state.data_ptr() + shift).to(torch.uint64)
sizes = torch.full((batch,), copy_size, dtype=torch.int32, device=device)
expected = snapshot.clone()
expected[:, :copy_size].copy_(snapshot[:, shift:])
for _ in range(10):
state.copy_(snapshot)
batch_memcpy(src_ptrs, dst_ptrs, sizes)
torch.accelerator.synchronize()
torch.testing.assert_close(state, expected, rtol=0, atol=0)
def test_matches_python_postprocess_mamba(self, device, test_config):
"""
Golden test: GPU kernel produces identical results to Python impl.
@@ -1190,12 +1219,27 @@ class TestPostprocessMambaFusedKernel:
# --- Verify Python behavior (ground truth) ---
dest_block_id = block_ids_per_req[0][1] # dest_block_idx = 1
# Conv state should be modified (shifted copy within block)
conv_changed = not torch.allclose(
conv_state_py[dest_block_id], conv_state_orig[dest_block_id]
# This is an overlapping in-place left shift, so comparing only the
# Python and fused paths can hide the same memcpy race in both. Build
# the memmove result from the untouched snapshot and check each path
# independently.
expected_conv_state = conv_state_orig.clone()
expected_conv_state[dest_block_id, :-1].copy_(
conv_state_orig[dest_block_id, 1:]
)
assert conv_changed, (
"Python: Conv state should be modified when accept_token_bias > 0"
torch.testing.assert_close(
conv_state_py,
expected_conv_state,
rtol=0,
atol=0,
msg="Python: overlapping conv copy should have memmove semantics",
)
torch.testing.assert_close(
conv_state_gpu,
expected_conv_state,
rtol=0,
atol=0,
msg="GPU: overlapping conv copy should have memmove semantics",
)
# Temporal state should be modified (copy from different block)
@@ -2122,13 +2166,10 @@ class TestPostprocessMambaFusedKernel:
actual_src_block_idx = src_block_idx + accept_token_bias
actual_src_block_id = block_table[req, actual_src_block_idx]
All prior regression tests exercise only ``bias == 1``, i.e. they
only ever read one slot ahead of ``src_block_idx`` in the block
table. An off-by-one (or missing scale) in the address computation
on line 143 of ``mamba_utils.py`` would be invisible to every
existing test but would silently read the wrong physical block on
any speculative-decode cycle that accepts multiple tokens across a
block boundary, feeding a stale hidden state forward one step.
A ``bias == 1`` case only reads one slot ahead of ``src_block_idx``
in the block table. This test isolates the larger-stride case, where
an off-by-one would read the wrong physical block after multiple
tokens are accepted across a block boundary.
Setup (block_size=16):
- running = 28 + 2 - 0 = 30
@@ -2141,8 +2182,7 @@ class TestPostprocessMambaFusedKernel:
With identity block_ids = [0,1,2,3,...], an off-by-one that used
bias=1 would copy from block_ids[2]=2 instead of block_ids[3]=3,
producing a clear state-value mismatch against the Python
reference.
producing a clear mismatch against the untouched snapshot.
"""
cfg = test_config
torch.manual_seed(7002)
@@ -2166,6 +2206,7 @@ class TestPostprocessMambaFusedKernel:
fwd_py,
fwd_gpu,
) = _make_dual_layer_state(cfg, device)
conv_state_orig = conv_state_py.clone()
temporal_state_orig = temporal_state_py.clone()
# --- Python reference ---
@@ -2212,12 +2253,22 @@ class TestPostprocessMambaFusedKernel:
device=device,
)
# --- Ground truth: Python must have sourced temporal from block 3 ---
# --- Ground truth from untouched snapshots ---
actual_src_block_id = block_ids_per_req[0][3] # == 3
dest_block_id = block_ids_per_req[0][1] # == 1
expected_conv_state = conv_state_orig.clone()
expected_conv_state[dest_block_id, :-2].copy_(
conv_state_orig[dest_block_id, 2:]
)
torch.testing.assert_close(conv_state_py, expected_conv_state, rtol=0, atol=0)
torch.testing.assert_close(conv_state_gpu, expected_conv_state, rtol=0, atol=0)
# Python must have sourced temporal from block 3.
torch.testing.assert_close(
temporal_state_py[dest_block_id],
temporal_state_orig[actual_src_block_id],
rtol=0,
atol=0,
msg=(
"Python reference did not copy from block_ids[src+bias]=3; "
"test preconditions are wrong"
@@ -2251,21 +2302,44 @@ class TestPostprocessMambaFusedKernel:
msg="num_accepted_tokens mismatch at accept_token_bias=2",
)
def test_ds_conv_layout_bias_gt_0_byte_equal_to_sd(
self, device, test_config, monkeypatch
@pytest.mark.parametrize(
"same_physical_block", [True, False], ids=["same", "distinct"]
)
@pytest.mark.parametrize("accept_token_bias", [1, 2, 3])
@pytest.mark.parametrize(
"dtype",
[torch.float16, torch.float32, torch.float64],
ids=["fp16", "fp32", "fp64"],
)
def test_sd_and_ds_conv_layouts_match_snapshot(
self,
device,
test_config,
monkeypatch,
accept_token_bias,
same_physical_block,
dtype,
):
"""DS conv postprocess should match SD when accept_token_bias > 0."""
"""SD and DS copies should independently match memmove semantics."""
from vllm.model_executor.layers.mamba import mamba_utils as model_mamba_utils
cfg = test_config
cfg.dtype = dtype
torch.manual_seed(38898)
req_ids = ["req_0"]
num_computed_tokens = [30]
num_scheduled_tokens = {"req_0": 1}
# Keep new_num_computed on an aligned boundary while varying how far
# below it the running state starts. This makes the copy bias exactly
# ``accept_token_bias`` for each case. The 32 boundary keeps source and
# destination in logical block 1; the 64 boundary copies block 2 -> 3.
aligned_boundary = 32 if same_physical_block else 64
num_computed_tokens = [aligned_boundary - 2 * accept_token_bias]
num_scheduled_tokens = {"req_0": accept_token_bias}
num_draft_tokens: dict[str, int] = {}
num_accepted_tokens = [2] # Results in accept_token_bias = 1
mamba_state_idx = [1] # src_block_idx = 1 = dest_block_idx
num_accepted_tokens = [accept_token_bias + 1]
dest_block_idx = aligned_boundary // cfg.block_size - 1
src_block_idx = dest_block_idx if same_physical_block else dest_block_idx - 1
mamba_state_idx = [src_block_idx]
block_ids_per_req = [list(range(8))]
layer_names = ["layer_0"]
@@ -2288,7 +2362,8 @@ class TestPostprocessMambaFusedKernel:
cfg.num_blocks, cfg.temporal_state_dim, dtype=cfg.dtype, device=device
)
# SD GPU path. Default layout is SD.
# SD GPU path.
monkeypatch.delenv("VLLM_SSM_CONV_STATE_LAYOUT", raising=False)
model_mamba_utils.get_conv_state_layout.cache_clear()
sd_conv = sd_source_conv.clone()
sd_temporal = sd_source_temporal.clone()
@@ -2312,9 +2387,32 @@ class TestPostprocessMambaFusedKernel:
)
torch.accelerator.synchronize()
# Sanity: SD path actually modified the state (copy was performed).
assert not torch.equal(sd_conv, sd_source_conv), (
"SD baseline did not modify conv state; test setup is wrong"
src_block_id = block_ids_per_req[0][src_block_idx]
dest_block_id = block_ids_per_req[0][dest_block_idx]
expected_conv = sd_source_conv.clone()
expected_conv[dest_block_id, :-accept_token_bias].copy_(
sd_source_conv[src_block_id, accept_token_bias:]
)
torch.testing.assert_close(
sd_conv,
expected_conv,
rtol=0,
atol=0,
msg="SD conv copy did not match the untouched source snapshot",
)
actual_temporal_src_idx = src_block_idx + accept_token_bias
actual_temporal_src_id = block_ids_per_req[0][actual_temporal_src_idx]
expected_temporal = sd_source_temporal.clone()
expected_temporal[dest_block_id].copy_(
sd_source_temporal[actual_temporal_src_id]
)
torch.testing.assert_close(
sd_temporal,
expected_temporal,
rtol=0,
atol=0,
msg="SD temporal copy did not match the untouched source snapshot",
)
# DS GPU path on the DS twin.
@@ -2346,22 +2444,39 @@ class TestPostprocessMambaFusedKernel:
# Reset the lru cache so other tests see the default layout again.
model_mamba_utils.get_conv_state_layout.cache_clear()
# DS bytes, un-permuted, should match the SD result.
# Validate DS independently against the snapshot; otherwise a shared
# SD/DS bug would remain invisible.
ds_conv_sd_layout = ds_conv.permute(0, 2, 1).contiguous()
torch.testing.assert_close(
ds_conv.permute(0, 2, 1).contiguous(),
sd_conv,
msg=(
"DS conv post-kernel does not match SD baseline; the DS "
"row-loop in postprocess_mamba_fused_kernel is wrong."
),
ds_conv_sd_layout,
expected_conv,
rtol=0,
atol=0,
msg="DS conv copy did not match the untouched source snapshot",
)
torch.testing.assert_close(
ds_temporal,
sd_temporal,
msg="DS temporal state diverged from SD",
expected_temporal,
rtol=0,
atol=0,
msg="DS temporal copy did not match the untouched source snapshot",
)
expected_accepted = 1 if same_physical_block else accept_token_bias + 1
expected_accepted_tensor = torch.tensor(
[expected_accepted], dtype=torch.int32, device=device
)
torch.testing.assert_close(
gpu_ctx_sd.num_accepted_tokens_out[:num_reqs],
expected_accepted_tensor,
rtol=0,
atol=0,
msg="SD num_accepted_tokens result is wrong",
)
torch.testing.assert_close(
gpu_ctx_ds.num_accepted_tokens_out[:num_reqs],
gpu_ctx_sd.num_accepted_tokens_out[:num_reqs],
msg="DS num_accepted_tokens diverged from SD",
expected_accepted_tensor,
rtol=0,
atol=0,
msg="DS num_accepted_tokens result is wrong",
)