[Kernel] Support DS Mamba tail copy for MTP align mode (#45473)

Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com>
Co-authored-by: Thomas Parnell <tom.parnell@gmail.com>
This commit is contained in:
sungsoo ha
2026-06-16 22:50:30 +00:00
committed by GitHub
co-authored by Thomas Parnell
parent 520828789c
commit 4bf699d310
3 changed files with 182 additions and 34 deletions
+115
View File
@@ -2132,3 +2132,118 @@ class TestPostprocessMambaFusedKernel:
expected_accepted,
msg="num_accepted_tokens mismatch at accept_token_bias=2",
)
def test_ds_conv_layout_bias_gt_0_byte_equal_to_sd(
self, device, test_config, monkeypatch
):
"""DS conv postprocess should match SD when accept_token_bias > 0."""
from vllm.model_executor.layers.mamba import mamba_utils as model_mamba_utils
cfg = test_config
torch.manual_seed(38898)
req_ids = ["req_0"]
num_computed_tokens = [30]
num_scheduled_tokens = {"req_0": 1}
num_draft_tokens: dict[str, int] = {}
num_accepted_tokens = [2] # Results in accept_token_bias = 1
mamba_state_idx = [1] # src_block_idx = 1 = dest_block_idx
block_ids_per_req = [list(range(8))]
layer_names = ["layer_0"]
kv_cache_config = _make_kv_cache_config(cfg, layer_names)
num_reqs = len(req_ids)
block_table_gpu = torch.zeros(num_reqs, 8, dtype=torch.int32, device=device)
block_table_gpu[0, :8] = torch.tensor(block_ids_per_req[0], dtype=torch.int32)
# Same logical conv state in SD and DS layouts.
sd_source_conv = torch.randn(
cfg.num_blocks,
cfg.conv_width,
cfg.conv_inner_dim,
dtype=cfg.dtype,
device=device,
)
ds_source_conv = sd_source_conv.permute(0, 2, 1).contiguous()
sd_source_temporal = torch.randn(
cfg.num_blocks, cfg.temporal_state_dim, dtype=cfg.dtype, device=device
)
# SD GPU path. Default layout is SD.
model_mamba_utils.get_conv_state_layout.cache_clear()
sd_conv = sd_source_conv.clone()
sd_temporal = sd_source_temporal.clone()
forward_context_sd = {
"layer_0": _make_mock_attention(sd_conv, sd_temporal),
}
gpu_ctx_sd = _make_gpu_ctx(cfg, kv_cache_config, device)
_run_gpu_postprocess(
gpu_ctx_sd,
kv_cache_config=kv_cache_config,
forward_context=forward_context_sd,
copy_funcs=_COPY_FUNCS,
block_table=block_table_gpu,
req_ids=req_ids,
num_accepted_tokens=num_accepted_tokens,
mamba_state_idx=mamba_state_idx,
num_scheduled_tokens=num_scheduled_tokens,
num_computed_tokens=num_computed_tokens,
num_draft_tokens=num_draft_tokens,
device=device,
)
torch.accelerator.synchronize()
# Sanity: SD path actually modified the state (copy was performed).
assert not torch.equal(sd_conv, sd_source_conv), (
"SD baseline did not modify conv state; test setup is wrong"
)
# DS GPU path on the DS twin.
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS")
model_mamba_utils.get_conv_state_layout.cache_clear()
try:
ds_conv = ds_source_conv.clone()
ds_temporal = sd_source_temporal.clone()
forward_context_ds = {
"layer_0": _make_mock_attention(ds_conv, ds_temporal),
}
gpu_ctx_ds = _make_gpu_ctx(cfg, kv_cache_config, device)
_run_gpu_postprocess(
gpu_ctx_ds,
kv_cache_config=kv_cache_config,
forward_context=forward_context_ds,
copy_funcs=_COPY_FUNCS,
block_table=block_table_gpu,
req_ids=req_ids,
num_accepted_tokens=num_accepted_tokens,
mamba_state_idx=mamba_state_idx,
num_scheduled_tokens=num_scheduled_tokens,
num_computed_tokens=num_computed_tokens,
num_draft_tokens=num_draft_tokens,
device=device,
)
torch.accelerator.synchronize()
finally:
# Reset the lru cache so other tests see the default layout again.
model_mamba_utils.get_conv_state_layout.cache_clear()
# DS bytes, un-permuted, should match the SD result.
torch.testing.assert_close(
ds_conv.permute(0, 2, 1).contiguous(),
sd_conv,
msg=(
"DS conv post-kernel does not match SD baseline; the DS "
"row-loop in postprocess_mamba_fused_kernel is wrong."
),
)
torch.testing.assert_close(
ds_temporal,
sd_temporal,
msg="DS temporal state diverged from SD",
)
torch.testing.assert_close(
gpu_ctx_ds.num_accepted_tokens_out[:num_reqs],
gpu_ctx_sd.num_accepted_tokens_out[:num_reqs],
msg="DS num_accepted_tokens diverged from SD",
)
@@ -303,18 +303,14 @@ def get_conv_copy_spec(
src_block_id = block_ids[cur_block_idx]
offset = num_accepted_tokens - 1
if is_conv_state_dim_first():
# DS layout: (num_blocks, dim, state_len) — state_len is last.
if offset > 0:
# Slicing along the last dim yields a non-contiguous view
# because features (dim) are strided by state_len.
raise NotImplementedError(
"DS conv state layout does not yet support speculative "
"decoding with mamba_cache_mode='align' "
"(num_accepted_tokens > 1)."
)
# DS offset > 0 is handled by the fused postprocess kernel.
assert offset == 0, (
"DS conv state with num_accepted_tokens > 1 must be handled by "
"the fused postprocess kernel, not get_conv_copy_spec"
)
src_state = state[src_block_id]
else:
# SD layout: (num_blocks, state_len, dim) — dim contiguous.
# SD layout: (num_blocks, state_len, dim), with dim contiguous.
src_state = state[src_block_id, offset:]
return MambaCopySpec(
start_addr=src_state.data_ptr(), num_elements=src_state.numel()
+61 -24
View File
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
get_conv_copy_spec,
get_temporal_copy_spec,
is_conv_state_dim_first,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
@@ -43,6 +44,9 @@ def postprocess_mamba_fused_kernel(
state_inner_sizes_ptr, # number of elements in inner dimensions
state_conv_widths_ptr, # conv width for conv states (0 for temporal)
state_group_indices_ptr, # maps state_idx to group index in block table
# DS conv row metadata. Zero keeps the single-region copy path.
state_dim_row_count_ptr, # int32: per-block dim row count for DS conv
state_dim_row_stride_ptr, # int64: bytes between rows for DS conv
# Output: num_accepted_tokens update (for src==dst case)
num_accepted_tokens_out_ptr,
# Runtime parameter (varies per batch - NOT constexpr to avoid recompilation)
@@ -52,6 +56,7 @@ def postprocess_mamba_fused_kernel(
block_size: tl.constexpr,
# COPY_BLOCK_SIZE: fixed tuning parameter for memory copy loop
COPY_BLOCK_SIZE: tl.constexpr,
CONV_STATE_DIM_FIRST: tl.constexpr,
):
"""
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
@@ -121,8 +126,35 @@ def postprocess_mamba_fused_kernel(
# conv_width == 0 means this is a temporal state (get_temporal_copy_spec logic)
is_conv_state = conv_width > 0
# Update accepted-token count before early exits.
if src_block_idx == dest_block_idx and state_idx == 0:
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
# Skip no-op self-copy.
if src_block_idx == dest_block_idx and accept_token_bias == 0:
return
if CONV_STATE_DIM_FIRST and is_conv_state:
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
per_row_bytes = (conv_width - accept_token_bias).to(tl.int64) * state_elem_size
bias_bytes = accept_token_bias.to(tl.int64) * state_elem_size
src_block_addr = state_base_addr + src_block_id * state_block_stride
dst_block_addr = state_base_addr + dest_block_id * state_block_stride
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for d in range(0, dim_rows):
row_src = src_block_addr + d * row_stride + bias_bytes
row_dst = dst_block_addr + d * row_stride
for i in range(0, per_row_bytes, COPY_BLOCK_SIZE):
mask = (i + offsets) < per_row_bytes
curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8))
curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8))
data = tl.load(curr_src, mask=mask)
tl.store(curr_dst, data, mask=mask)
return
if is_conv_state:
# Conv state: copy
# SD conv: copy
# state[block_table[req_idx, src_block_idx], accept_token_bias:]
# to
# state[block_table[req_idx, dest_block_idx], :conv_width - accept_token_bias]
@@ -151,19 +183,6 @@ def postprocess_mamba_fused_kernel(
# actual data when the state tensor uses as_strided page padding.
copy_size = state_inner_size * state_elem_size
# Mirror postprocess_mamba's trailing
# if src_block_idx == dest_block_idx: num_accepted_tokens_cpu[i] = 1
# This runs whether or not the copy below is skipped (it's per-request, so
# only state_idx == 0 writes).
if src_block_idx == dest_block_idx and state_idx == 0:
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
# Mirror collect_mamba_copy_meta's early return: src==dst with no token
# bias means source and destination ranges coincide, so the copy is a
# no-op.
if src_block_idx == dest_block_idx and accept_token_bias == 0:
return
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for i in range(0, copy_size, COPY_BLOCK_SIZE):
mask = (i + offsets) < copy_size
@@ -271,6 +290,9 @@ class MambaSpecDecodeGPUContext:
state_inner_sizes: torch.Tensor # int64: elements in inner dimensions
state_conv_widths: torch.Tensor # int32: conv width (0 for temporal states)
state_group_indices: torch.Tensor # int32: maps state_idx to group index
# DS conv row metadata. Zero keeps the single-region copy path.
state_dim_row_count: torch.Tensor # int32: per-block dim row count
state_dim_row_stride: torch.Tensor # int64: bytes between rows
# Configuration
block_size: int
@@ -338,6 +360,12 @@ class MambaSpecDecodeGPUContext:
state_group_indices=torch.zeros(
total_states, dtype=torch.int32, device=device
),
state_dim_row_count=torch.zeros(
total_states, dtype=torch.int32, device=device
),
state_dim_row_stride=torch.zeros(
total_states, dtype=torch.int64, device=device
),
block_size=mamba_spec.block_size,
num_layers=num_layers,
num_state_types=num_state_types,
@@ -430,17 +458,23 @@ class MambaSpecDecodeGPUContext:
or copy_func is get_temporal_copy_spec
), f"unexpected copy func: {copy_func}"
if copy_func is get_conv_copy_spec:
# Conv state: conv_width is state.size(1)
# inner_size is stride(1) = elements per conv position,
# used to compute byte offset for state[block, offset:]
conv_w = state.size(1) if state.dim() > 1 else 0
self.state_conv_widths[idx] = conv_w
if state.dim() > 2:
# stride(1) = product of dims[2:] for contiguous tensor
self.state_inner_sizes[idx] = state.stride(1)
else:
# 2D tensor: [num_blocks, conv_dim], no inner dims
if state.dim() != 3:
raise ValueError(
"Expected 3D conv state cache, got "
f"shape {tuple(state.shape)}"
)
if is_conv_state_dim_first():
# DS layout: state_len is the slide axis.
self.state_conv_widths[idx] = state.size(2)
self.state_inner_sizes[idx] = 1
self.state_dim_row_count[idx] = state.size(1)
self.state_dim_row_stride[idx] = (
state.stride(1) * state.element_size()
)
else:
# SD layout: dim is contiguous.
self.state_conv_widths[idx] = state.size(1)
self.state_inner_sizes[idx] = state.stride(1)
else:
# Temporal state: inner_size = natural elements per
# block (prod of inner dims). The kernel uses this
@@ -521,10 +555,13 @@ class MambaSpecDecodeGPUContext:
self.state_inner_sizes,
self.state_conv_widths,
self.state_group_indices,
self.state_dim_row_count,
self.state_dim_row_stride,
self.num_accepted_tokens_out,
num_reqs,
block_size=self.block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
)