[Kernel] Support DS Mamba tail copy for MTP align mode (#45473)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Co-authored-by: Thomas Parnell <tom.parnell@gmail.com>
This commit is contained in:
co-authored by
Thomas Parnell
parent
520828789c
commit
4bf699d310
@@ -2132,3 +2132,118 @@ class TestPostprocessMambaFusedKernel:
|
||||
expected_accepted,
|
||||
msg="num_accepted_tokens mismatch at accept_token_bias=2",
|
||||
)
|
||||
|
||||
def test_ds_conv_layout_bias_gt_0_byte_equal_to_sd(
|
||||
self, device, test_config, monkeypatch
|
||||
):
|
||||
"""DS conv postprocess should match SD when accept_token_bias > 0."""
|
||||
from vllm.model_executor.layers.mamba import mamba_utils as model_mamba_utils
|
||||
|
||||
cfg = test_config
|
||||
torch.manual_seed(38898)
|
||||
|
||||
req_ids = ["req_0"]
|
||||
num_computed_tokens = [30]
|
||||
num_scheduled_tokens = {"req_0": 1}
|
||||
num_draft_tokens: dict[str, int] = {}
|
||||
num_accepted_tokens = [2] # Results in accept_token_bias = 1
|
||||
mamba_state_idx = [1] # src_block_idx = 1 = dest_block_idx
|
||||
block_ids_per_req = [list(range(8))]
|
||||
|
||||
layer_names = ["layer_0"]
|
||||
kv_cache_config = _make_kv_cache_config(cfg, layer_names)
|
||||
|
||||
num_reqs = len(req_ids)
|
||||
block_table_gpu = torch.zeros(num_reqs, 8, dtype=torch.int32, device=device)
|
||||
block_table_gpu[0, :8] = torch.tensor(block_ids_per_req[0], dtype=torch.int32)
|
||||
|
||||
# Same logical conv state in SD and DS layouts.
|
||||
sd_source_conv = torch.randn(
|
||||
cfg.num_blocks,
|
||||
cfg.conv_width,
|
||||
cfg.conv_inner_dim,
|
||||
dtype=cfg.dtype,
|
||||
device=device,
|
||||
)
|
||||
ds_source_conv = sd_source_conv.permute(0, 2, 1).contiguous()
|
||||
sd_source_temporal = torch.randn(
|
||||
cfg.num_blocks, cfg.temporal_state_dim, dtype=cfg.dtype, device=device
|
||||
)
|
||||
|
||||
# SD GPU path. Default layout is SD.
|
||||
model_mamba_utils.get_conv_state_layout.cache_clear()
|
||||
sd_conv = sd_source_conv.clone()
|
||||
sd_temporal = sd_source_temporal.clone()
|
||||
forward_context_sd = {
|
||||
"layer_0": _make_mock_attention(sd_conv, sd_temporal),
|
||||
}
|
||||
gpu_ctx_sd = _make_gpu_ctx(cfg, kv_cache_config, device)
|
||||
_run_gpu_postprocess(
|
||||
gpu_ctx_sd,
|
||||
kv_cache_config=kv_cache_config,
|
||||
forward_context=forward_context_sd,
|
||||
copy_funcs=_COPY_FUNCS,
|
||||
block_table=block_table_gpu,
|
||||
req_ids=req_ids,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
mamba_state_idx=mamba_state_idx,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
device=device,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Sanity: SD path actually modified the state (copy was performed).
|
||||
assert not torch.equal(sd_conv, sd_source_conv), (
|
||||
"SD baseline did not modify conv state; test setup is wrong"
|
||||
)
|
||||
|
||||
# DS GPU path on the DS twin.
|
||||
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS")
|
||||
model_mamba_utils.get_conv_state_layout.cache_clear()
|
||||
try:
|
||||
ds_conv = ds_source_conv.clone()
|
||||
ds_temporal = sd_source_temporal.clone()
|
||||
forward_context_ds = {
|
||||
"layer_0": _make_mock_attention(ds_conv, ds_temporal),
|
||||
}
|
||||
gpu_ctx_ds = _make_gpu_ctx(cfg, kv_cache_config, device)
|
||||
_run_gpu_postprocess(
|
||||
gpu_ctx_ds,
|
||||
kv_cache_config=kv_cache_config,
|
||||
forward_context=forward_context_ds,
|
||||
copy_funcs=_COPY_FUNCS,
|
||||
block_table=block_table_gpu,
|
||||
req_ids=req_ids,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
mamba_state_idx=mamba_state_idx,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
device=device,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
finally:
|
||||
# Reset the lru cache so other tests see the default layout again.
|
||||
model_mamba_utils.get_conv_state_layout.cache_clear()
|
||||
|
||||
# DS bytes, un-permuted, should match the SD result.
|
||||
torch.testing.assert_close(
|
||||
ds_conv.permute(0, 2, 1).contiguous(),
|
||||
sd_conv,
|
||||
msg=(
|
||||
"DS conv post-kernel does not match SD baseline; the DS "
|
||||
"row-loop in postprocess_mamba_fused_kernel is wrong."
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ds_temporal,
|
||||
sd_temporal,
|
||||
msg="DS temporal state diverged from SD",
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
gpu_ctx_ds.num_accepted_tokens_out[:num_reqs],
|
||||
gpu_ctx_sd.num_accepted_tokens_out[:num_reqs],
|
||||
msg="DS num_accepted_tokens diverged from SD",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user