[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
File diff suppressed because it is too large Load Diff
-472
View File
@@ -1,472 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
import pytest
import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (
current_platform.is_cuda_alike()
or current_platform.is_xpu()
or current_platform.is_cpu()
),
reason="causal_conv1d Triton kernels require CUDA-alike, XPU, or CPU",
)
def causal_conv1d_ref(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None = None,
initial_states: torch.Tensor | None = None,
return_final_states: bool = False,
final_states_out: torch.Tensor | None = None,
activation: str | None = "silu",
):
"""x: (batch, dim, seqlen)
weight: (dim, width)
bias: (dim,)
initial_states: (batch, dim, width - 1)
final_states_out: (batch, dim, width - 1)
out: (batch, dim, seqlen)
"""
if activation not in [None, "silu", "swish"]:
raise NotImplementedError("activation must be None, silu, or swish")
dtype_in = x.dtype
x = x.to(weight.dtype)
seqlen = x.shape[-1]
dim, width = weight.shape
if initial_states is None:
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim)
else:
x = torch.cat([initial_states, x], dim=-1)
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim)
out = out[..., :seqlen]
if return_final_states:
final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to(
dtype_in
) # (batch, dim, width - 1)
if final_states_out is not None:
final_states_out.copy_(final_states)
else:
final_states_out = final_states
out = (out if activation is None else F.silu(out)).to(dtype=dtype_in)
return (out, None) if not return_final_states else (out, final_states_out)
def causal_conv1d_update_ref(
x, conv_state, weight, bias=None, activation=None, cache_seqlens=None
):
"""x: (batch, dim) or (batch, dim, seqlen)
conv_state: (batch, dim, state_len), where state_len >= width - 1
weight: (dim, width)
bias: (dim,)
cache_seqlens: (batch,), dtype int32.
If not None, the conv_state is treated as a circular buffer.
The conv_state will be updated by copying x to the
conv_state starting at the index
@cache_seqlens % state_len before performing the convolution.
out: (batch, dim) or (batch, dim, seqlen)
"""
if activation not in [None, "silu", "swish"]:
raise NotImplementedError("activation must be None, silu, or swish")
dtype_in = x.dtype
unsqueeze = x.dim() == 2
if unsqueeze:
x = x.unsqueeze(-1)
batch, dim, seqlen = x.shape
width = weight.shape[1]
state_len = conv_state.shape[-1]
assert conv_state.shape == (batch, dim, state_len)
assert weight.shape == (dim, width)
if cache_seqlens is None:
x_new = torch.cat([conv_state, x], dim=-1).to(
weight.dtype
) # (batch, dim, state_len + seqlen)
conv_state.copy_(x_new[:, :, -state_len:])
else:
width_idx = torch.arange(
-(width - 1), 0, dtype=torch.long, device=x.device
).unsqueeze(0) + cache_seqlens.unsqueeze(1)
width_idx = (
torch.remainder(width_idx, state_len).unsqueeze(1).expand(-1, dim, -1)
)
x_new = torch.cat([conv_state.gather(2, width_idx), x], dim=-1).to(weight.dtype)
copy_idx = torch.arange(seqlen, dtype=torch.long, device=x.device).unsqueeze(
0
) + cache_seqlens.unsqueeze(1)
copy_idx = torch.remainder(copy_idx, state_len).unsqueeze(1).expand(-1, dim, -1)
conv_state.scatter_(2, copy_idx, x)
out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0, groups=dim)[
:, :, -seqlen:
]
if unsqueeze:
out = out.squeeze(-1)
return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [False, True])
@pytest.mark.parametrize("has_bias", [False, True])
@pytest.mark.parametrize("seqlen", [1])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype):
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
# set seed
set_random_seed(0)
batch = 2
x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
x_ref = x.clone()
# +1 entry to reserve index 0 as null block
conv_state = torch.randn(batch + 1, dim, width - 1, device=device, dtype=itype)
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
# Start indices from 1, skipping null block at index 0
conv_state_indices = torch.arange(1, batch + 1, dtype=torch.int32, device=device)
conv_state_ref = conv_state[conv_state_indices].detach().clone()
activation = None if not silu_activation else "silu"
out = causal_conv1d_update(
x,
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=conv_state_indices,
)
out_ref = causal_conv1d_update_ref(
x_ref, conv_state_ref, weight, bias, activation=activation
)
assert torch.equal(conv_state[conv_state_indices], conv_state_ref)
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [False, True])
@pytest.mark.parametrize("has_bias", [False, True])
@pytest.mark.parametrize("seqlen", [1, 3])
@pytest.mark.parametrize("width", [3, 4])
@pytest.mark.parametrize("dim", [2048 + 16, 4096])
# tests correctness in case subset of the sequences are padded
@pytest.mark.parametrize("with_padding", [True, False])
@pytest.mark.parametrize("batch_size", [3])
def test_causal_conv1d_update_with_batch_gather(
batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype
):
device = DEVICE
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
# set seed
set_random_seed(0)
padding = 5 if with_padding else 0
padded_batch_size = batch_size + padding
# total_entries = number of cache line
total_entries = 10 * batch_size
# x will be (batch, dim, seqlen) with contiguous along dim-axis
x = torch.randn(
padded_batch_size, seqlen, dim, device=device, dtype=itype
).transpose(1, 2)
x_ref = x.clone()
# +1 to exclude index 0 (null block)
conv_state_indices = (torch.randperm(total_entries - 1)[:batch_size] + 1).to(
dtype=torch.int32, device=device
)
unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device)
unused_states_bool[conv_state_indices] = False
padded_state_indices = torch.concat(
[
conv_state_indices,
torch.as_tensor(
[NULL_BLOCK_ID] * padding, dtype=torch.int32, device=device
),
],
dim=0,
)
# conv_state will be (cache_lines, dim, state_len)
# with contiguous along dim-axis
conv_state = torch.randn(
total_entries, width - 1, dim, device=device, dtype=itype
).transpose(1, 2)
conv_state_for_padding_test = conv_state.clone()
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
conv_state_ref = conv_state[conv_state_indices, :].detach().clone()
activation = None if not silu_activation else "silu"
out = causal_conv1d_update(
x,
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=padded_state_indices,
)
out_ref = causal_conv1d_update_ref(
x_ref[:batch_size], conv_state_ref, weight, bias, activation=activation
)
assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
assert torch.equal(
conv_state[unused_states_bool], conv_state_for_padding_test[unused_states_bool]
)
assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [True])
@pytest.mark.parametrize("has_bias", [True])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("seqlen", [8, 249, 4096])
@pytest.mark.parametrize("dim", [64, 4096])
@pytest.mark.parametrize("with_padding", [True, False])
@pytest.mark.parametrize("batch", [4, 10])
def test_causal_conv1d_varlen(
batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype
):
device = DEVICE
if not current_platform.is_cpu():
torch.accelerator.empty_cache()
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
if itype == torch.bfloat16:
rtol, atol = 1e-2, 5e-2
# set seed
set_random_seed(0)
seqlens = []
batch_size = batch
padding = 3 if with_padding else 0
padded_batch_size = batch_size + padding
nsplits = padded_batch_size - 1
eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
seqlens.append(
torch.diff(
torch.cat([torch.tensor([-1]), eos_pos, torch.tensor([seqlen - 1])])
).tolist()
)
assert sum(seqlens[-1]) == seqlen
assert all(s > 0 for s in seqlens[-1])
total_entries = batch_size * 10
cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum], dim=0)
x = rearrange(
torch.randn(1, seqlen, 4096 + dim + 64, device=device, dtype=itype),
"b s d -> b d s",
)[:, 4096 : 4096 + dim, :]
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
x_ref = x.clone()
weight_ref = weight.clone()
bias_ref = bias.clone() if bias is not None else None
activation = None if not silu_activation else "silu"
final_states = torch.randn(
total_entries, width - 1, dim, device=x.device, dtype=x.dtype
).transpose(1, 2)
final_states_ref = final_states.clone()
has_initial_states = torch.randint(
0, 2, (cumsum.shape[0] - 1,), dtype=torch.bool, device=x.device
)
# +1 to exclude index 0 (null block)
state_indices = (
torch.randperm(total_entries - 1, dtype=torch.int32, device=x.device)[
:batch_size
]
+ 1
)
padded_state_indices = torch.concat(
[
state_indices,
torch.as_tensor(
[NULL_BLOCK_ID] * padding, dtype=torch.int32, device=device
),
],
dim=-1,
)
out = causal_conv1d_fn(
x.squeeze(0),
weight,
bias=bias,
conv_states=final_states,
query_start_loc=cumsum.to(device),
cache_indices=padded_state_indices,
has_initial_state=has_initial_states,
activation=activation,
)
out_ref = []
out_ref_b = []
splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)]
for i in range(len(seqlens[0])):
x_s = [v[i].unsqueeze(0) for v in splits][0]
if padded_state_indices[i] == NULL_BLOCK_ID:
continue
out_ref_b.append(
causal_conv1d_ref(
x_s,
weight_ref,
bias_ref,
activation=activation,
return_final_states=True,
final_states_out=final_states_ref[padded_state_indices[i]].unsqueeze(0),
initial_states=final_states_ref[padded_state_indices[i]].unsqueeze(0)
if has_initial_states[i]
else None,
)
)
out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2))
out_ref_tensor = torch.cat(out_ref, dim=0)
assert torch.allclose(
final_states[state_indices],
final_states_ref[state_indices],
rtol=rtol,
atol=atol,
)
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [True])
@pytest.mark.parametrize("has_bias", [True])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [64, 2048])
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
@pytest.mark.skipif(
current_platform.is_cpu(),
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
)
def test_causal_conv1d_update_ragged_varlen(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
mk_inputs = lambda: _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
)
inputs, state = mk_inputs()
actual = causal_conv1d_update(conv_state=state, **inputs)
ref_inputs, ref_state = mk_inputs()
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
for i, ref_out in enumerate(ref_outs):
start = int(inputs["query_start_loc"][i])
end = int(inputs["query_start_loc"][i + 1])
entry = inputs["conv_state_indices"][i : i + 1]
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
assert torch.equal(state[entry], ref_state[entry])
def _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
device = DEVICE
set_random_seed(0)
batch_size = len(q_lens_cfg)
max_query_len = 5
assert max(q_lens_cfg) <= max_query_len
state_len = width - 1 + max_query_len
total_entries = 10 * batch_size
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
]
)
total_tokens = int(q_lens.sum())
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
conv_state = torch.randn(
total_entries, state_len, dim, device=device, dtype=itype
).transpose(1, 2)
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
activation = None if not silu_activation else "silu"
conv_state_indices = (
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
).to(torch.int32)
num_accepted_tokens = torch.randint(
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
)
inputs = dict(
x=x,
weight=weight,
bias=bias,
activation=activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=max_query_len,
)
return inputs, conv_state
def _create_ragged_varlen_ref_outs(
x,
conv_state,
weight,
bias,
activation,
conv_state_indices,
num_accepted_tokens,
query_start_loc,
max_query_len,
) -> Iterator[torch.Tensor]:
for seq_idx in range(len(conv_state_indices)):
start = int(query_start_loc[seq_idx])
end = int(query_start_loc[seq_idx + 1])
entry = conv_state_indices[seq_idx : seq_idx + 1]
if start == end:
yield x[start:end]
continue
out_ref = causal_conv1d_update(
x[start:end],
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=entry,
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
query_start_loc=query_start_loc.new_tensor([0, end - start]),
max_query_len=max_query_len,
)
yield out_ref
-179
View File
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.mamba.short_conv import ShortConv
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
from vllm.platforms import current_platform
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
@pytest.fixture(autouse=True)
def mock_dist():
with (
patch(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
return_value=1,
),
patch(
"vllm.distributed.parallel_state.model_parallel_is_initialized",
return_value=True,
),
patch(
"vllm.distributed.parallel_state.get_tp_group",
return_value=MagicMock(rank_in_group=0),
),
):
yield
@pytest.fixture
def vllm_config():
# ShortConv only needs compilation_config from the current vLLM config, so a
# minimal config (model_config=None) avoids mocking ModelConfig and the
# associated VllmConfig validation churn.
return VllmConfig(compilation_config=CompilationConfig())
def test_short_conv_forward_native_prefill(vllm_config):
prefix = "test_layer"
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
dim = 16
from vllm.config import set_current_vllm_config
with set_current_vllm_config(vllm_config):
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
layer.to("cpu")
# vLLM Linear layers allocate weights with torch.empty (uninitialized).
# On ARM these come back as zero-filled pages, so in_proj output is zero and
# the prefill state stays zero. Seed + init to make the test platform-safe.
torch.manual_seed(0)
for p in layer.parameters():
torch.nn.init.normal_(p)
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
# Mock AttentionMetadata
num_prefills = 1
num_prefill_tokens = 5
query_start_loc_p = torch.tensor([0, 5], dtype=torch.int32)
state_indices_tensor_p = torch.tensor([0], dtype=torch.int32)
# ShortConvAttentionMetadata
attn_metadata = ShortConvAttentionMetadata(
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
num_decodes=0,
num_decode_tokens=0,
num_reqs=1,
query_start_loc_p=query_start_loc_p,
has_initial_states_p=torch.tensor([False]),
state_indices_tensor_p=state_indices_tensor_p,
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
seq_lens=torch.tensor([5]),
)
# Mock KV cache
# conv_state shape (num_blocks, L_cache - 1, dim)
conv_state = torch.zeros((1, config.conv_L_cache - 1, dim))
layer.kv_cache = (conv_state,)
hidden_states = torch.randn((num_prefill_tokens, dim))
output = torch.zeros_like(hidden_states)
attn_metadata_dict = {prefix: attn_metadata}
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
layer.forward_native(hidden_states, output)
# Check if KV cache was updated
assert not torch.allclose(conv_state, torch.zeros_like(conv_state))
def test_short_conv_forward_native_decode(vllm_config):
prefix = "test_layer_decode"
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
dim = 16
from vllm.config import set_current_vllm_config
with set_current_vllm_config(vllm_config):
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
layer.to("cpu")
torch.manual_seed(0)
for p in layer.parameters():
torch.nn.init.normal_(p)
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
# Mock AttentionMetadata for 2 decode requests
num_decodes = 2
state_indices_tensor_d = torch.tensor([0, 1], dtype=torch.int32)
attn_metadata = ShortConvAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decodes,
num_reqs=num_decodes,
query_start_loc_p=None,
has_initial_states_p=None,
state_indices_tensor_p=torch.empty((0,), dtype=torch.int32),
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
seq_lens=torch.tensor([1, 1]),
)
# Mock KV cache (2 blocks for 2 requests)
conv_state = torch.randn((2, config.conv_L_cache - 1, dim))
layer.kv_cache = (conv_state,)
hidden_states = torch.randn((num_decodes, dim))
output = torch.zeros_like(hidden_states)
old_conv_state = conv_state.clone()
attn_metadata_dict = {prefix: attn_metadata}
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
layer.forward_native(hidden_states, output)
# Check if KV cache was updated
assert not torch.allclose(conv_state, old_conv_state)
def test_dispatch_cpu_unquantized_gemm_conv_layer():
# Convolution layers have >2D weights; dispatch should skip them gracefully.
# Shape/dtype are AMX-pack safe (bf16, width==4, dim % block_size == 0) so
# the AMX prepack branch does not raise on AMX-capable CPUs.
class MockConvLayer(torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.nn.Parameter(
torch.randn(32, 1, 4, dtype=torch.bfloat16)
)
self.bias = torch.nn.Parameter(torch.randn(32, dtype=torch.bfloat16))
layer = MockConvLayer()
# The ndim != 2 guard returns early without raising.
dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
# No cpu_linear set — conv layers are handled elsewhere.
assert not hasattr(layer, "cpu_linear")
-392
View File
@@ -1,392 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-path tests for fused Qwen3.5 GDN MTP decode."""
from __future__ import annotations
import types
from typing import cast
from unittest.mock import patch
import pytest
import torch
from vllm.platforms import current_platform
if not (current_platform.is_cuda() and current_platform.has_device_capability(80)):
pytest.skip(
reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.",
allow_module_level=True,
)
from tests.v1.attention.utils import ( # noqa: E402
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
ChunkGatedDeltaRule,
QwenGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
MambaStateShapeCalculator,
)
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402
rmsnorm_fn,
)
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
NUM_SPEC = 3
SPEC_TOKENS = NUM_SPEC + 1
H = 1
HV = 8
K = 128
V = 128
CONV_KERNEL = 4
CONV_DIM = 2 * H * K + HV * V
BLOCK_SIZE = 16
PREFIX = "model.layers.0.linear_attn"
EPS = 1e-6
class _TestGatedNorm:
def __init__(self, weight: torch.Tensor, activation: str) -> None:
self.weight = weight
self.bias = None
self.eps = EPS
self.group_size = None
self.norm_before_gate = True
self.activation = activation
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
return rmsnorm_fn(
x,
self.weight,
None,
z=z,
eps=self.eps,
norm_before_gate=True,
activation=self.activation,
)
def _make_vllm_config():
config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
hf_config_override={"linear_key_head_dim": K},
)
config.additional_config = {"gdn_prefill_backend": "cutedsl"}
config.cache_config.mamba_cache_mode = "none"
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=NUM_SPEC
)
return config
def _build_layer(
vllm_config,
conv_state: torch.Tensor,
ssm_state: torch.Tensor,
a_log: torch.Tensor,
dt_bias: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor,
output_gate_activation: str,
):
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_packed_recurrent_decode=False,
disable_tp_for_ba_proj=False,
tp_size=1,
num_k_heads=H,
num_v_heads=HV,
head_k_dim=K,
head_v_dim=V,
key_dim=K,
value_dim=HV * V,
activation="silu",
A_log=a_log,
dt_bias=dt_bias,
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
kv_cache=(conv_state, ssm_state),
norm=_TestGatedNorm(norm_weight, output_gate_activation),
layer_norm_epsilon=EPS,
gdn_decode_kernel="cuda",
)
with set_current_vllm_config(vllm_config):
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
for name in (
"rearrange_mixed_qkv",
"_forward_core",
"_forward_core_decode_spec_post_conv_fused_norm",
"_forward_core_decode_spec_fused_norm",
"_can_use_fused_gdn_mtp_decode",
"_rms_norm_gated_cuda",
"_forward_core_fused_norm",
"_forward_core_fused_norm_packed",
"split_ba",
):
setattr(
layer,
name,
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
)
return layer
@pytest.mark.parametrize(
"num_v_heads,expected",
[
pytest.param(2, True, id="ratio1"),
pytest.param(4, True, id="ratio2"),
pytest.param(6, True, id="ratio3"),
pytest.param(8, True, id="ratio4"),
pytest.param(16, True, id="ratio8"),
pytest.param(5, False, id="non-integral-ratio"),
pytest.param(10, False, id="unsupported-ratio5"),
],
)
def test_fused_mtp_head_ratio_guard(num_v_heads: int, expected: bool) -> None:
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
pytest.skip("fused GDN decode MTP op is not built")
layer = types.SimpleNamespace(
num_k_heads=2,
num_v_heads=num_v_heads,
kv_cache=(None, torch.empty(1, dtype=torch.float32, device="cuda")),
gdn_decode_kernel="cuda",
)
attn_metadata = types.SimpleNamespace(
spec_state_indices_tensor=torch.ones(
1, SPEC_TOKENS, dtype=torch.int32, device="cuda"
),
spec_sequence_masks=object(),
num_decodes=0,
num_spec_decodes=1,
)
assert (
QwenGatedDeltaNetAttention._can_use_fused_gdn_mtp_decode(
cast(QwenGatedDeltaNetAttention, layer),
cast(GDNAttentionMetadata, attn_metadata),
)
is expected
)
@torch.inference_mode()
def test_fused_forward_uses_packed_entrypoint() -> None:
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
device = torch.device("cuda")
num_tokens = 3
hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device)
mixed_qkvz = torch.randn(
num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device
)
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_fused_gdn_spec_decode=True,
norm=types.SimpleNamespace(
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
),
num_v_heads=HV,
tp_size=1,
head_v_dim=V,
in_proj_qkvz=lambda _: (mixed_qkvz, None),
in_proj_ba=lambda _: (ba, None),
out_proj=lambda x: (x, None),
)
layer.forward_cuda = types.MethodType(
QwenGatedDeltaNetAttention.forward_cuda, layer
)
def packed_op(
actual_qkvz: torch.Tensor,
actual_ba: torch.Tensor,
output: torch.Tensor,
*,
layer_name: str,
) -> None:
assert actual_qkvz is mixed_qkvz
assert actual_ba is ba
assert layer_name == _encode_layer_name(PREFIX)
output.fill_(1)
with (
patch.object(
torch.ops.vllm,
"qwen_gdn_attention_core_fused_norm_packed",
side_effect=packed_op,
) as packed_mock,
patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock,
):
output = layer.forward_cuda(hidden_states)
packed_mock.assert_called_once()
triton_mock.assert_not_called()
torch.testing.assert_close(output, torch.ones_like(output))
@pytest.mark.parametrize(
"seq_lens,query_lens,draft_tokens,expected_fused_calls",
[
pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"),
pytest.param(
[128, 96],
[SPEC_TOKENS, 64],
[NUM_SPEC, -1],
0,
id="mixed-mtp-falls-back",
),
pytest.param([96], [64], [-1], 0, id="pure-prefill"),
pytest.param([128], [1], [-1], 0, id="pure-decode"),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_model_path_matches_reference(
seq_lens: list[int],
query_lens: list[int],
draft_tokens: list[int],
expected_fused_calls: int,
output_gate_activation: str,
) -> None:
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
torch.manual_seed(1)
device = torch.device("cuda")
vllm_config = _make_vllm_config()
builder = GDNAttentionMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
),
layer_names=[PREFIX],
vllm_config=vllm_config,
device=device,
)
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
common = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
common.block_table_tensor.add_(1)
with set_current_vllm_config(vllm_config):
metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common,
num_accepted_tokens=torch.ones(
batch.batch_size, dtype=torch.int32, device=device
),
num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32),
)
state_indices = [
indices
for indices in (
metadata.spec_state_indices_tensor,
metadata.non_spec_state_indices_tensor,
)
if indices is not None and indices.numel() > 0
]
pool_size = max(int(indices.max().item()) for indices in state_indices) + 1
conv_state_shape, temporal_state_shape = (
MambaStateShapeCalculator.gated_delta_net_state_shape(
1, H, HV, K, V, CONV_KERNEL, NUM_SPEC
)
)
conv_state_seed = 0.05 * torch.randn(
pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device
)
ssm_state_seed = 0.01 * torch.randn(
pool_size, *temporal_state_shape, dtype=torch.float32, device=device
)
a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
conv_weight = 0.1 * torch.randn(
CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device
)
norm_weight = torch.randn(V, dtype=torch.float32, device=device)
num_tokens = batch.compute_num_tokens()
mixed_qkv = 0.1 * torch.randn(
num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device
)
b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device)
a = 0.1 * torch.randn_like(b)
output_gate = 0.1 * torch.randn(
num_tokens, HV, V, dtype=torch.bfloat16, device=device
)
mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1)
ba = torch.cat((b, a), dim=-1)
context = types.SimpleNamespace(attn_metadata={PREFIX: metadata})
reference_layer = _build_layer(
vllm_config,
conv_state_seed.clone(),
ssm_state_seed.clone(),
a_log,
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
reference_out = torch.zeros_like(output_gate)
with patch.object(
qwen_gdn_linear_attn, "get_forward_context", return_value=context
):
reference_layer._forward_core(
mixed_qkv=mixed_qkv.clone(),
b=b,
a=a,
core_attn_out=reference_out,
)
reference_out = reference_layer.norm(reference_out, output_gate)
fused_layer = _build_layer(
vllm_config,
conv_state_seed.clone(),
ssm_state_seed.clone(),
a_log,
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
context.no_compile_layers = {PREFIX: fused_layer}
fused_out = torch.zeros_like(output_gate)
fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp
with (
patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context),
patch.object(
qwen_gdn_linear_attn.ops,
"fused_gdn_decode_post_conv_mtp",
wraps=fused_op,
) as fused_mock,
):
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz.clone(),
ba,
fused_out,
layer_name=_encode_layer_name(PREFIX),
)
assert fused_mock.call_count == expected_fused_calls
torch.testing.assert_close(
fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0
)
torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(
fused_layer.kv_cache[1],
reference_layer.kv_cache[1],
atol=3e-2,
rtol=3e-2,
)
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_rocm():
pytest.skip(
reason="FlashInfer GDN prefill is not supported on ROCm.",
allow_module_level=True,
)
import flashinfer.gdn_prefill # noqa: E402
import torch.nn.functional as F # noqa: E402
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
fi_chunk_gated_delta_rule,
) # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
captured_cu_seqlens = None
def fake_chunk_gated_delta_rule(**kwargs):
nonlocal captured_cu_seqlens
captured_cu_seqlens = kwargs["cu_seqlens"]
return kwargs["q"]
monkeypatch.setattr(
flashinfer.gdn_prefill,
"chunk_gated_delta_rule",
fake_chunk_gated_delta_rule,
)
q = torch.zeros(1, 2, 1, 2)
cu_seqlens = torch.tensor([0, 2], dtype=torch.int32)
output, final_state = fi_chunk_gated_delta_rule(
q=q,
k=q,
v=q,
g=torch.zeros(1, 2, 1),
beta=torch.zeros(1, 2, 1),
initial_state=torch.zeros(1, 1, 2, 2),
output_final_state=False,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
)
assert captured_cu_seqlens is not None
assert captured_cu_seqlens.dtype == torch.int64
assert output.shape == q.shape
assert final_state is None
@pytest.mark.skipif(
not (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
),
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
)
@pytest.mark.parametrize("use_cp", [True, False])
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
"""Each exported checkpoint is the state of the prefill stopped there.
The leading decode row is outside the prefill kernel. The two checkpointed
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
so the exporter must pick each state out of the shared emitted buffer;
the third prefill takes no checkpoint. Both FlashInfer routes are covered
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
slots after the window, which the conv kernels leave alone.
"""
torch.manual_seed(0)
device = "cuda"
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
num_spec = 2
offsets = [0, 128, 0, 192]
query_lens = [200, 50, 300]
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
num_blocks = 8
num_tokens = sum(query_lens)
q = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
k = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
initial_state = 0.1 * torch.randn(
len(query_lens), num_v_heads, head_dim, head_dim, device=device
)
prefill_cu = [0]
for query_len in query_lens:
prefill_cu.append(prefill_cu[-1] + query_len)
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
state_checkpoints = torch.empty(
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
)
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=torch.tensor(prefill_cu, device=device),
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
checkpoint_every_n_tokens=every_n_tokens,
use_cp=use_cp,
)
# The exporter sees the decode row's token in front of the prefills.
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
conv_state = torch.zeros(
num_blocks,
state_len + num_spec,
conv_dim,
device=device,
dtype=torch.bfloat16,
).transpose(-1, -2)
ssm_state = torch.zeros(
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
)
MambaPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
state_indices=torch.tensor(state_indices, device=device),
),
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=cu_seqlens,
recurrent_rows=torch.tensor(rows, device=device),
)
written = []
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
if not offset:
continue
written.append(block)
seq = row - 1
start = prefill_cu[seq]
_, expected_state = fi_chunk_gated_delta_rule(
q=q[:, start : start + offset],
k=k[:, start : start + offset],
v=v[:, start : start + offset],
g=g[:, start : start + offset],
beta=beta[:, start : start + offset],
initial_state=initial_state[seq : seq + 1],
output_final_state=True,
cu_seqlens=torch.tensor([0, offset], device=device),
use_qk_l2norm_in_kernel=False,
)
torch.testing.assert_close(
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
)
end = cu_seqlens[row].item() + offset
torch.testing.assert_close(
conv_state[block, :, :state_len],
raw_qkv[end - state_len : end].T,
atol=0,
rtol=0,
)
assert not conv_state[block, :, state_len:].any()
untouched = [i for i in range(num_blocks) if i not in written]
assert not ssm_state[untouched].any()
assert not conv_state[untouched].any()
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layout dispatch tests for ``QwenGatedDeltaNetAttention._forward_core_rocm``.
The AITER fused reshape+conv kernel learned Qwen3.5's flat ``[q|k|v|z]``
packing in https://github.com/ROCm/aiter/pull/3251, so flat-layout models take
the decode fast path too instead of falling back to the generic path.
The dispatch tests run host-side with recording stubs. The prefill numerical
tests exercise the production ChunkGatedDeltaRule wrapper on ROCm against a
token-wise Torch reference, including ragged state and output-buffer layouts.
"""
from __future__ import annotations
import types
from unittest.mock import patch
import pytest
import torch
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
QwenGatedDeltaNetAttention,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
PREFIX = "model.layers.0.linear_attn"
H = 2 # num key heads
HV = 4 # num value heads
K = 8 # head_k_dim
V = 8 # head_v_dim
def _make_metadata(
*,
num_prefills: int = 0,
num_decodes: int = 2,
spec_sequence_masks: torch.Tensor | None = None,
):
"""Only the fields the dispatch condition reads carry meaningful values."""
return GDNAttentionMetadata(
num_prefills=num_prefills,
num_prefill_tokens=num_prefills,
num_decodes=num_decodes,
num_decode_tokens=num_decodes,
num_spec_decodes=0,
num_spec_decode_tokens=0,
num_actual_tokens=num_prefills + num_decodes,
spec_sequence_masks=spec_sequence_masks,
)
def _make_layer(gqa_interleaved_layout: bool):
"""Stub layer running the real ``_forward_core_rocm``, recording dispatch."""
layer = types.SimpleNamespace()
layer.prefix = PREFIX
layer.gqa_interleaved_layout = gqa_interleaved_layout
layer.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat"
layer.calls = []
layer._forward_core_decode_aiter = lambda **kw: layer.calls.append("aiter")
layer._forward_core = lambda **kw: layer.calls.append("generic")
layer.prepare_gdn_attention_core_inputs = lambda qkvz, ba, n: (
torch.zeros(n, H * K * 2 + HV * V),
torch.zeros(n, HV, V),
torch.zeros(n, HV),
torch.zeros(n, HV),
)
layer._forward_core_rocm = types.MethodType(
QwenGatedDeltaNetAttention._forward_core_rocm, layer
)
return layer
def _run(layer, meta) -> str:
num_tokens = max(meta.num_actual_tokens, 1)
ctx = types.SimpleNamespace(attn_metadata={PREFIX: meta})
with patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=ctx):
layer._forward_core_rocm(
qkvz=torch.zeros(num_tokens, 2 * H * K + 2 * HV * V),
ba=torch.zeros(num_tokens, 2 * HV),
z_out=torch.zeros(num_tokens, HV, V),
core_attn_out=torch.zeros(num_tokens, HV, V),
)
assert len(layer.calls) == 1
return layer.calls[0]
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
def test_pure_decode_takes_the_fast_path(gqa_interleaved_layout: bool) -> None:
"""Both packings reach the fast path; flat used to be guarded out."""
layer = _make_layer(gqa_interleaved_layout)
assert _run(layer, _make_metadata()) == "aiter"
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
def test_layout_string_matches_the_packing(gqa_interleaved_layout: bool) -> None:
"""The value handed to the kernel's ``qkvz_layout`` parameter."""
layer = _make_layer(gqa_interleaved_layout)
expected = "interleaved" if gqa_interleaved_layout else "flat"
assert layer.qkvz_layout == expected
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
@pytest.mark.parametrize(
"meta_kwargs",
[
{"num_prefills": 1},
{"num_decodes": 0},
{"spec_sequence_masks": torch.zeros(1, dtype=torch.bool)},
],
ids=["has_prefill", "no_decode", "spec_decode"],
)
def test_non_pure_decode_batches_use_the_generic_path(
gqa_interleaved_layout: bool, meta_kwargs: dict
) -> None:
layer = _make_layer(gqa_interleaved_layout)
assert _run(layer, _make_metadata(**meta_kwargs)) == "generic"
def _prefill_reference(q, k, v, g, beta, initial_state, lengths):
"""Independent token-wise recurrence in the layer's [value, key] layout."""
q, k, v, g, beta, initial_state = (
x.cpu().float() for x in (q, k, v, g, beta, initial_state)
)
q = q.repeat_interleave(v.shape[2] // q.shape[2], dim=2)
k = k.repeat_interleave(v.shape[2] // k.shape[2], dim=2)
q = q * q.shape[-1] ** -0.5
output = torch.empty_like(v)
final_state = torch.empty_like(initial_state)
previous_threads = torch.get_num_threads()
torch.set_num_threads(1)
try:
start = 0
for sequence, length in enumerate(lengths):
state = initial_state[sequence].clone()
for token in range(start, start + length):
state = state * g[0, token].exp()[:, None, None]
key = k[0, token]
prediction = (state * key[:, None, :]).sum(dim=-1)
delta = (v[0, token] - prediction) * beta[0, token, :, None]
state = state + delta[:, :, None] * key[:, None, :]
output[0, token] = (state * q[0, token, :, None, :]).sum(dim=-1)
final_state[sequence] = state
start += length
finally:
torch.set_num_threads(previous_threads)
return output, final_state
@pytest.mark.parametrize(
"state_dtype,correlated",
[(torch.bfloat16, False), (torch.float32, False), (torch.float32, True)],
ids=["ragged-bf16-state", "ragged-fp32-state", "correlated-keys"],
)
@torch.inference_mode()
def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
ChunkGatedDeltaRule,
)
from vllm.third_party.flash_linear_attention.ops.index import (
prepare_chunk_indices,
prepare_chunk_offsets,
)
if not torch.cuda.is_available():
pytest.skip("ROCm prefill numerics require a GPU")
# Only the config is needed for production backend selection; no weights.
config = VllmConfig(
model_config=ModelConfig(
model="Qwen/Qwen3.5-0.8B",
revision="2fc06364715b967f1860aea9cf38778875588b17",
max_model_len=1024,
),
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
additional_config={"gdn_prefill_backend": "triton"},
)
with set_current_vllm_config(config):
prefill = ChunkGatedDeltaRule()
assert prefill.gdn_prefill_backend == "triton"
assert prefill._forward_method == prefill.forward_native
torch.manual_seed(1234)
lengths = [64] if correlated else [1, 63, 64, 65, 129]
tokens, key_heads, value_heads, dim = sum(lengths), 4, 8, 128
q = torch.randn(1, tokens, key_heads, dim, device="cuda")
q = torch.nn.functional.normalize(q, dim=-1).to(torch.bfloat16)
k = torch.randn_like(q)
k = torch.nn.functional.normalize(k.float(), dim=-1).to(q.dtype)
v = torch.randn(1, tokens, value_heads, dim, device="cuda", dtype=q.dtype)
g = -torch.rand(1, tokens, value_heads, device="cuda") * 0.1
beta = torch.rand_like(g)
state = (torch.randn(len(lengths), value_heads, dim, dim, device="cuda") * 0.05).to(
state_dtype
)
if correlated:
q.zero_()
k.zero_()
q[..., 0] = 1
k[..., 0] = 1
g.zero_()
beta.fill_(0.9)
state.zero_()
expected, expected_state = _prefill_reference(q, k, v, g, beta, state, lengths)
initial_state_copy = state.clone()
cu_seqlens = torch.tensor(
[0, *torch.tensor(lengths).cumsum(0).tolist()],
device="cuda",
dtype=torch.int32,
)
kwargs = dict(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=state,
output_final_state=True,
cu_seqlens=cu_seqlens,
chunk_indices=prepare_chunk_indices(cu_seqlens, 64),
chunk_offsets=prepare_chunk_offsets(cu_seqlens, 64),
use_qk_l2norm_in_kernel=False,
)
storage = torch.full(
(tokens + 3, value_heads, dim), float("nan"), device="cuda", dtype=q.dtype
)
output, final_state = prefill(**kwargs, core_attn_out=storage[:tokens])
assert output.data_ptr() == storage.data_ptr()
assert output.shape == v.shape
assert final_state.shape == state.shape
assert torch.isnan(storage[tokens:]).all()
torch.testing.assert_close(state, initial_state_copy, atol=0, rtol=0)
# Preserve the upstream prefill suite's numerical budgets. The reference
# here is a token-wise recurrence, independent of the chunk implementation.
output_error = (output.cpu().float() - expected).abs()
state_error = (final_state.cpu().float() - expected_state).abs()
assert torch.isfinite(output).all() and torch.isfinite(final_state).all()
assert output_error.max() < (1e-1 if correlated else 2e-3)
assert output_error.mean() < (2e-3 if correlated else 6e-5)
assert state_error.max() < (2e-1 if correlated else 2e-2)
assert state_error.mean() < 6e-4
no_buffer_output, no_buffer_state = prefill(**kwargs)
torch.testing.assert_close(no_buffer_output, output, atol=0, rtol=0)
torch.testing.assert_close(no_buffer_state, final_state, atol=0, rtol=0)
@@ -1,169 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""An SSD intermediate state equals an independent run over the same prefix.
This is the central claim of internal prefill checkpoints: the state the SSD
scan already materializes at the checkpoint's chunk boundary is exactly the
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
cached and later resumed from. The last test covers the other half: moving
that state, and the conv window, into the paged cache.
"""
import pytest
import torch
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
store_prefill_checkpoint,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
state when a checkpoint was requested, else the per-sequence final state.
"""
A, dt, X, B, C = inputs
offsets = [checkpoint_offset] if checkpoint_offset else None
cu, seq_idx, last, ckpt = compute_chunk_metadata(
chunk_size,
1,
torch.tensor([0]),
torch.tensor([0, seqlen]),
checkpoint_offsets_p=offsets,
)
x = X[:seqlen]
out = torch.empty_like(x)
states = mamba_chunk_scan_combined_varlen(
x,
dt[:seqlen],
A,
B[:seqlen],
C[:seqlen],
chunk_size,
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
out=out,
D=None,
return_intermediate_states=checkpoint_offset > 0,
state_dtype=torch.float32,
)
return states, ckpt[0]
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
48 is deliberately not a multiple of chunk_size: align-mode block sizes
carry no chunk_size factor, so off-grid checkpoints are the common case
and are what the chunk split exists to support.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
checkpoint_state = states[ckpt_idx]
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
torch.testing.assert_close(
checkpoint_state.to(torch.float32),
prefix_final[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
def test_checkpoint_does_not_perturb_the_final_state():
"""Exporting a checkpoint must not change what the prefill produces.
The chunk split is an extra boundary, not a different computation, so the
sequence's final state has to match a run that took no checkpoint.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
torch.testing.assert_close(
with_ckpt[-1].to(torch.float32),
without_ckpt[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
@pytest.mark.parametrize("spec_slots", [0, 3])
@pytest.mark.parametrize("dim_first", [True, False])
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
"""The store reads the checkpoint row out of every chunk state.
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
program's own row, so the store only passes if ``chunk_idx`` is used.
Row 1 declines and must write nothing. The conv input is a column slice of
a wider projection, the SD conv layout is a transposed view, and SSM rows
are padded pages, matching the strides the mixer passes. Spec decoding
widens the conv state, and the store must still fill only the first
``state_len`` slots with the newest inputs.
"""
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
projection = torch.randn(64, dim + 8, device=DEVICE)
conv_input = projection[:, :dim]
num_slots = state_len + spec_slots
if dim_first:
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
else:
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
# Blocks are pages holding more than the SSM state, so rows are padded.
row = heads * head_dim * d_state
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
checkpoint = MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
state_indices=torch.tensor(
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
),
offsets=[32, 0],
)
store_prefill_checkpoint(
checkpoint,
torch.tensor([3, 0], device=DEVICE),
conv_input.T,
conv_state,
varlen_states,
ssm_state,
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
state_len=state_len,
)
exact = dict(rtol=0, atol=0)
torch.testing.assert_close(
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
)
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
assert not conv_state[:2].any() and not ssm_storage[:2].any()
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
File diff suppressed because it is too large Load Diff
-627
View File
@@ -1,627 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from einops import rearrange, repeat
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
_chunk_scan_fwd,
_chunk_scan_fwd_kernel,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
# All kernels exercised here are pure Triton, so they run on any backend
# that the vLLM platform layer treats as a CUDA-alike device or as XPU.
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Mamba2 SSD Triton kernels require a CUDA-alike or XPU device.",
)
# Added by the IBM Team, 2024
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py
# this is the segsum implementation taken from above
def segsum(x):
"""Calculates segment sum."""
T = x.size(-1)
x = repeat(x, "... d -> ... d e", e=T)
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=-1)
x = x.masked_fill(~mask, 0)
x_segsum = torch.cumsum(x, dim=-2)
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=0)
x_segsum = x_segsum.masked_fill(~mask, -torch.inf)
return x_segsum
def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
"""Arguments:
X: (batch, length, n_heads, d_head)
A: (batch, length, n_heads)
B: (batch, length, n_heads, d_state)
C: (batch, length, n_heads, d_state)
Return:
Y: (batch, length, n_heads, d_head)
"""
assert X.dtype == A.dtype == B.dtype == C.dtype
assert X.shape[1] % block_len == 0
# Rearrange into blocks/chunks
X, A, B, C = (
rearrange(x, "b (c l) ... -> b c l ...", l=block_len) for x in (X, A, B, C)
)
A = rearrange(A, "b c l h -> b h c l")
A_cumsum = torch.cumsum(A, dim=-1)
# 1. Compute the output for each intra-chunk (diagonal blocks)
L = torch.exp(segsum(A))
Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, X)
# 2. Compute the state for each intra-chunk
# (right term of low-rank factorization of off-diagonal blocks; B terms)
decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum)
states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, X)
# 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at
# chunk boundaries
# (middle term of factorization of off-diag blocks; A terms)
if initial_states is None:
initial_states = torch.zeros_like(states[:, :1])
states = torch.cat([initial_states, states], dim=1)
decay_chunk = torch.exp(segsum(F.pad(A_cumsum[:, :, :, -1], (1, 0))))
new_states = torch.einsum("bhzc,bchpn->bzhpn", decay_chunk, states)
states, final_state = new_states[:, :-1], new_states[:, -1]
# 4. Compute state -> output conversion per chunk
# (left term of low-rank factorization of off-diagonal blocks; C terms)
state_decay_out = torch.exp(A_cumsum)
Y_off = torch.einsum("bclhn,bchpn,bhcl->bclhp", C, states, state_decay_out)
# Add output of intra-chunk and inter-chunk terms
# (diagonal and off-diagonal blocks)
Y = rearrange(Y_diag + Y_off, "b c l h p -> b (c l) h p")
return Y, final_state
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device=DEVICE):
set_random_seed(0)
A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device))
dt = F.softplus(
torch.randn(batch_size, seqlen, n_heads, dtype=itype, device=device) - 4
)
X = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
B = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
C = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
return A, dt, X, B, C
def generate_continuous_batched_examples(
example_lens_by_batch,
num_examples,
full_length,
last_taken,
exhausted,
n_heads,
d_head,
itype,
device=DEVICE,
return_naive_ref=True,
):
# this function generates a random examples of certain length
# and then cut according to "example_lens_by_batch" and feed
# them in continuous batches to the kernels.
# If if return_naive_ref=True, the naive torch implementation
# ssd_minimal_discrete will be used to compute and return
# reference output.
# generate the full-length example
A, dt, X, B, C = generate_random_inputs(
num_examples, full_length, n_heads, d_head, itype
)
if return_naive_ref:
Y_min, final_state_min = ssd_minimal_discrete(
X * dt.unsqueeze(-1), A * dt, B, C, block_len=full_length // 4
)
# internal function that outputs a cont batch of examples
# given a tuple of lengths for each example in the batch
# e.g., example_lens=(8, 4) means take 8 samples from first eg,
# 4 examples from second eg, etc
def get_continuous_batch(example_lens: tuple[int, ...]):
indices = []
for i, x in enumerate(example_lens):
c = last_taken.get(i, 0)
indices.append((c, c + x))
last_taken[i] = (c + x) % full_length
exhausted[i] = last_taken[i] == 0
return (
torch.concat([x[i, s:e] for i, (s, e) in enumerate(indices)]).unsqueeze(0)
for x in (dt, X, B, C)
)
# internal function that maps "n" to the appropriate right boundary
# value when forming continuous batches from examples of length given
# by "full_length".
# - e.g., when n > full_length, returns n % full_length
# when n == full_length, returns full_length
def end_boundary(n: int):
return n - ((n - 1) // full_length) * full_length
IND_E = None
for spec in example_lens_by_batch:
# get the (maybe partial) example seen in this cont batch
dt2, X2, B2, C2 = get_continuous_batch(spec)
# get the metadata
cu_seqlens = torch.tensor((0,) + spec, device=device).cumsum(dim=0)
seq_idx = torch.zeros(
cu_seqlens[-1], dtype=torch.int32, device=cu_seqlens.device
)
for i, (srt, end) in enumerate(
zip(
cu_seqlens,
cu_seqlens[1:],
)
):
seq_idx[srt:end] = i
# for cont batch
if IND_E is None:
IND_S = [0 for _ in range(len(spec))]
else:
IND_S = [x % full_length for x in IND_E]
IND_E = [end_boundary(x + y) for x, y in zip(IND_S, spec)]
# varlen has implicit batch=1
dt2 = dt2.squeeze(0)
X2 = X2.squeeze(0)
B2 = B2.squeeze(0)
C2 = C2.squeeze(0)
yield (
[Y_min[s, IND_S[s] : IND_E[s]] for s in range(num_examples)]
if return_naive_ref
else None,
cu_seqlens,
seq_idx,
(A, dt2, X2, B2, C2),
)
@pytest.mark.parametrize("itype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("n_heads", [4, 16, 32])
@pytest.mark.parametrize("d_head", [5, 8, 32, 128])
@pytest.mark.parametrize("seq_len_chunk_size", [(112, 16), (128, 32)])
def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, itype):
# this tests the kernels on a single example (bs=1)
# TODO: the bfloat16 case requires higher thresholds. To be investigated
if itype == torch.bfloat16:
atol, rtol = 5e-2, 5e-2
else:
atol, rtol = 8e-3, 5e-3
# set seed
batch_size = 1 # batch_size
# ssd_minimal_discrete requires chunk_size divide seqlen
# - this is only required for generating the reference seqs,
# it is not an operational limitation.
seqlen, chunk_size = seq_len_chunk_size
A, dt, X, B, C = generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype)
Y_min, final_state_min = ssd_minimal_discrete(
X * dt.unsqueeze(-1), A * dt, B, C, chunk_size
)
cu_seqlens = torch.tensor((0, seqlen), device=DEVICE).cumsum(dim=0)
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
)
# varlen has implicit batch=1
X = X.squeeze(0)
dt = dt.squeeze(0)
A = A.squeeze(0)
B = B.squeeze(0)
C = C.squeeze(0)
Y = torch.empty_like(X)
final_state = mamba_chunk_scan_combined_varlen(
X,
dt,
A,
B,
C,
chunk_size,
cu_seqlens=cu_seqlens.to(torch.int32),
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx_chunks,
out=Y,
D=None,
)
# just test the last in sequence
torch.testing.assert_close(Y[-1], Y_min[0, -1], atol=atol, rtol=rtol)
# just test the last head
# NOTE, in the kernel we always cast states to fp32
torch.testing.assert_close(
final_state[:, -1].to(torch.float32),
final_state_min[:, -1].to(torch.float32),
atol=atol,
rtol=rtol,
)
@pytest.mark.parametrize("itype", [torch.float32])
@pytest.mark.parametrize("n_heads", [4, 8])
@pytest.mark.parametrize("d_head", [5, 16, 32])
@pytest.mark.parametrize(
"seq_len_chunk_size_cases",
[
# small-ish chunk_size (8)
(64, 8, 2, [(64, 32), (64, 32)]),
(64, 8, 2, [(8, 8), (8, 8), (8, 8)]), # chunk size boundary
(
64,
8,
2,
[(4, 4), (4, 4), (4, 4), (4, 4)],
), # chunk_size larger than cont batches
(64, 8, 5, [(64, 32, 16, 8, 8)]),
# large-ish chunk_size (256)
(64, 256, 1, [(5,), (1,), (1,), (1,)]), # irregular sizes with small sequences
(
64,
256,
2,
[(5, 30), (1, 2), (1, 2), (1, 2)],
), # irregular sizes with small sequences
# we also need to test some large seqlen
# to catch errors with init states decay
(768, 128, 2, [(138, 225), (138, 225)]),
],
)
def test_mamba_chunk_scan_cont_batch(d_head, n_heads, seq_len_chunk_size_cases, itype):
# this test with multiple examples in a continuous batch
# (i.e. chunked prefill)
seqlen, chunk_size, num_examples, cases = seq_len_chunk_size_cases
# This test can have larger error for longer sequences
if seqlen > 256:
atol, rtol = 1e-2, 5e-3
else:
atol, rtol = 5e-3, 5e-3
# hold state during the cutting process so we know if an
# example has been exhausted and needs to cycle
last_taken: dict = {} # map: eg -> pointer to last taken sample
exhausted: dict = {} # map: eg -> boolean indicating example is exhausted
states = None
for Y_min, cu_seqlens, _token_seq_idx, (
A,
dt,
X,
B,
C,
) in generate_continuous_batched_examples(
cases, num_examples, seqlen, last_taken, exhausted, n_heads, d_head, itype
):
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
)
Y = torch.empty_like(X)
new_states = mamba_chunk_scan_combined_varlen(
X,
dt,
A,
B,
C,
chunk_size,
cu_seqlens=cu_seqlens.to(torch.int32),
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx_chunks,
out=Y,
D=None,
initial_states=states,
)
# just test the last in sequence
for i in range(num_examples):
# just test one dim and dstate
Y_eg = Y[cu_seqlens[i] : cu_seqlens[i + 1], 0, 0]
Y_min_eg = Y_min[i][:, 0, 0]
torch.testing.assert_close(Y_eg, Y_min_eg, atol=atol, rtol=rtol)
# update states
states = new_states
for i, clear in exhausted.items():
if clear:
states[i].fill_(0.0)
exhausted[i] = False
@pytest.mark.parametrize("chunk_size", [8, 256])
@pytest.mark.parametrize(
"seqlens",
[(16, 20), (270, 88, 212, 203)],
)
def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
# This test verifies the correctness of the chunked prefill implementation
# in the mamba2 ssd kernels, by comparing concatenation (in the sequence
# dimension) of chunked results with the full sequence result.
# It is different from test_mamba_chunk_scan_cont_batch by:
# 1. Not using the naive torch implementation (ssd_minimal_discrete) to get
# reference outputs. Instead, it compares chunked kernel outputs to full
# sequence kernel outputs. This is the most straightforward way to
# assert chunked prefill correctness.
# 2. It focuses on cases where sequences change in the middle of mamba
# chunks, and not necessarily on chunk boundaries.
max_seqlen = max(seqlens)
# This test can have larger error for longer sequences
if max_seqlen > 256:
atol, rtol = 1e-2, 5e-3
else:
atol, rtol = 5e-3, 5e-3
num_sequences = len(seqlens)
n_heads = 16
d_head = 64
itype = torch.float32
# hold state during the cutting process so we know if an
# example has been exhausted and needs to cycle
last_taken: dict = {} # map: eg -> pointer to last taken sample
exhausted: dict = {} # map: eg -> boolean indicating example is exhausted
_, cu_seqlens, seq_idx, (A, dt, X, B, C) = next(
generate_continuous_batched_examples(
[seqlens],
num_sequences,
max_seqlen,
last_taken,
exhausted,
n_heads,
d_head,
itype,
return_naive_ref=False,
)
)
seqlens = torch.tensor(seqlens, dtype=torch.int32, device=X.device)
device = X.device
## full seqlen computation
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
)
Y_ref = torch.empty_like(X)
state_ref = mamba_chunk_scan_combined_varlen(
X,
dt,
A,
B,
C,
chunk_size,
cu_seqlens=cu_seqlens.to(torch.int32),
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx_chunks,
out=Y_ref,
D=None,
initial_states=None,
)
## chunked seqlen computation
# first chunk
chunked_seqlens = seqlens // 2
chunked_cu_seqlens = torch.cat(
[torch.tensor([0], device=device), torch.cumsum(chunked_seqlens, dim=0)], dim=0
)
chunked_input_seq_len = chunked_cu_seqlens[-1]
X_chunked = torch.zeros_like(X)[:chunked_input_seq_len, ...]
dt_chunked = torch.zeros_like(dt)[:chunked_input_seq_len, ...]
B_chunked = torch.zeros_like(B)[:chunked_input_seq_len, ...]
C_chunked = torch.zeros_like(C)[:chunked_input_seq_len, ...]
for i in range(num_sequences):
chunk_f = lambda x, i: x[
cu_seqlens[i] : cu_seqlens[i] + chunked_seqlens[i], ...
]
X_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
X, i
)
dt_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
dt, i
)
B_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
B, i
)
C_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
C, i
)
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(chunked_cu_seqlens, chunk_size)
)
Y_partial = torch.empty_like(X_chunked)
partial_state = mamba_chunk_scan_combined_varlen(
X_chunked,
dt_chunked,
A,
B_chunked,
C_chunked,
chunk_size,
cu_seqlens=chunked_cu_seqlens.to(torch.int32),
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx_chunks,
out=Y_partial,
D=None,
initial_states=None,
)
# remaining chunk
remaining_chunked_seqlens = seqlens - chunked_seqlens
remaining_chunked_cu_seqlens = torch.cat(
[
torch.tensor([0], device=device),
torch.cumsum(remaining_chunked_seqlens, dim=0),
],
dim=0,
)
remaining_chunked_input_seq_len = remaining_chunked_cu_seqlens[-1]
remaining_X_chunked = torch.zeros_like(X)[:remaining_chunked_input_seq_len, ...]
remaining_dt_chunked = torch.zeros_like(dt)[:remaining_chunked_input_seq_len, ...]
remaining_B_chunked = torch.zeros_like(B)[:remaining_chunked_input_seq_len, ...]
remaining_C_chunked = torch.zeros_like(C)[:remaining_chunked_input_seq_len, ...]
for i in range(num_sequences):
remaining_chunk_f = lambda x, i: x[
cu_seqlens[i] + chunked_seqlens[i] : cu_seqlens[i + 1], ...
]
remaining_X_chunked[
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
] = remaining_chunk_f(X, i)
remaining_dt_chunked[
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
] = remaining_chunk_f(dt, i)
remaining_B_chunked[
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
] = remaining_chunk_f(B, i)
remaining_C_chunked[
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
] = remaining_chunk_f(C, i)
# assert input chunking is correct
concat_chunk_f = lambda pt1, pt2, i: torch.cat(
[
pt1[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...],
pt2[
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1],
...,
],
],
dim=0,
)
concat_batch_f = lambda pt1, pt2: torch.cat(
[concat_chunk_f(pt1, pt2, i) for i in range(num_sequences)], dim=0
)
assert concat_batch_f(X_chunked, remaining_X_chunked).equal(X)
assert concat_batch_f(dt_chunked, remaining_dt_chunked).equal(dt)
assert concat_batch_f(B_chunked, remaining_B_chunked).equal(B)
assert concat_batch_f(C_chunked, remaining_C_chunked).equal(C)
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
compute_varlen_chunk_metadata(remaining_chunked_cu_seqlens, chunk_size)
)
Y_chunked = torch.empty_like(remaining_X_chunked)
state_chunked = mamba_chunk_scan_combined_varlen(
remaining_X_chunked,
remaining_dt_chunked,
A,
remaining_B_chunked,
remaining_C_chunked,
chunk_size,
cu_seqlens=remaining_chunked_cu_seqlens.to(torch.int32),
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx_chunks,
out=Y_chunked,
D=None,
initial_states=partial_state,
)
Y = concat_batch_f(Y_partial, Y_chunked)
# kernel chunked is same as kernel overall
for i in range(num_sequences):
Y_seq = Y[cu_seqlens[i] : cu_seqlens[i + 1], ...]
Y_ref_seq = Y_ref[cu_seqlens[i] : cu_seqlens[i + 1], ...]
torch.testing.assert_close(
Y_seq[: chunked_seqlens[i], ...],
Y_ref_seq[: chunked_seqlens[i], ...],
atol=atol,
rtol=rtol,
msg=lambda x, i=i: f"seq{i} output part1 " + x,
)
torch.testing.assert_close(
Y_seq[chunked_seqlens[i] :, ...],
Y_ref_seq[chunked_seqlens[i] :, ...],
atol=atol,
rtol=rtol,
msg=lambda x, i=i: f"seq{i} output part2 " + x,
)
state_seq = state_chunked[i]
state_seq_ref = state_ref[i]
torch.testing.assert_close(
state_seq,
state_seq_ref,
atol=atol,
rtol=rtol,
msg=lambda x, i=i: f"seq{i} state " + x,
)
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
"""The autotuner picks a config by speed only, so every config in the list
must give the same finite output. On sm90, configs that pipeline a wgmma
with `cb` in registers wrote NaN into whole rows (#61603)."""
from triton.runtime.errors import OutOfResources
set_random_seed(0)
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
seqlen, nchunks = 181, 2
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
ref = None
for config in list(_chunk_scan_fwd_kernel.configs):
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
for _ in range(10):
out = torch.empty_like(x)
try:
_chunk_scan_fwd(
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
)
except OutOfResources:
# The autotuner skips configs that do not fit this device.
break
assert torch.isfinite(out).all(), f"non-finite output with {config}"
if ref is None:
ref = out
torch.testing.assert_close(
out,
ref,
atol=2e-2,
rtol=2e-2,
msg=lambda m, config=config: f"{config}: {m}",
)
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for ``_memcpy_u64_tiled``.
Guards the head/body/tail alignment handling and the u64 body's tile
partitioning used by ``postprocess_mamba_fused_kernel`` and
``precopy_mamba_align_fused_kernel``. The device function must be
byte-identical to a ``dst[dst_off:dst_off+copy_size] = src[src_off:src_off+
copy_size]`` slice copy for every combination of:
* ``copy_size``: the degenerate 0-byte case, sub-8B (head-only), 8B-aligned
bodies, and a multi-tile body that spans several ``COPY_BLOCK_SIZE``
iterations.
* ``(src_off, dst_off)`` pairs: aligned, shared sub-8B misalignment (fast
path exercising head/tail masking), and mismatched src/dst alignment
(byte-fallback path). The kernel branches on ``(src ^ dst) & 7``, so a
full 5x5 product would re-exercise the same two codepaths.
* ``NUM_TILES``: 1 (SD conv callsite, single-CTA memcpy) and
``_TEMPORAL_TILES`` (temporal callsite, u64 range partitioned across
CTAs).
* ``COPY_BLOCK_SIZE``: the production value 1024 (single-tile at these
sizes) and a small value 8 so the existing ``copy_size`` cases actually
cross tile boundaries under ``NUM_TILES=_TEMPORAL_TILES``. The
partitioning math is COPY_BLOCK_SIZE-agnostic, so testing at a small
value covers boundary arithmetic for all values.
"""
from __future__ import annotations
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, _memcpy_u64_tiled
try:
import pytest
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="_memcpy_u64_tiled needs CUDA/Triton",
)
_parametrize = pytest.mark.parametrize
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
pytest = None
def _parametrize(_name, _values):
def _deco(fn):
return fn
return _deco
@triton.jit
def _memcpy_wrapper_kernel(
src_ptr,
dst_ptr,
copy_size,
src_off,
dst_off,
COPY_BLOCK_SIZE: tl.constexpr,
NUM_TILES: tl.constexpr,
):
"""Thin harness: one CTA per tile, exercising the device function."""
tile_idx = tl.program_id(0)
src_addr = src_ptr.to(tl.int64) + src_off
dst_addr = dst_ptr.to(tl.int64) + dst_off
_memcpy_u64_tiled(
src_addr,
dst_addr,
copy_size,
tile_idx,
COPY_BLOCK_SIZE=COPY_BLOCK_SIZE,
NUM_TILES=NUM_TILES,
)
# Copy sizes: 0/1/7 (all-head), 8 (all-body when dst-aligned), 15 (body+tail,
# tail-mask path), 16 (small tiled body), 1024 and 4 KiB (spans all 16 tiles
# at COPY_BLOCK_SIZE=8, with 4 KiB adding multi-iteration-per-tile coverage;
# both single-tile at COPY_BLOCK_SIZE=1024).
_COPY_SIZES = [0, 1, 7, 8, 15, 16, 1024, 4 * 1024]
# (src_off, dst_off) pairs. Torch tensors are 256B-aligned at data_ptr, so
# slicing by these bytes yields a controlled sub-8B alignment. The kernel
# branches on ``(src ^ dst) & 7``, so we cover both sides plus the aligned
# baseline; the full 5x5 product added no coverage.
_ALIGN_PAIRS = [
(0, 0), # fully aligned; head/tail masked out
(3, 3), # shared misalignment; fast path with head_bytes=5
(7, 7), # shared, extreme; head_bytes=1 with non-empty body
(1, 3), # mismatched; byte load/store fallback
(3, 1), # mismatched, opposite direction
]
_MAX_ALIGN_OFF = max(o for pair in _ALIGN_PAIRS for o in pair)
# (NUM_TILES, COPY_BLOCK_SIZE) configs. NUM_TILES=1 is the SD conv callsite
# (single-CTA memcpy) where tile partitioning collapses, so COPY_BLOCK_SIZE
# is not observable — one value suffices. NUM_TILES=_TEMPORAL_TILES is the
# temporal callsite: 1024 matches the production launch; 8 shrinks per-tile
# rounding so ``_COPY_SIZES``' 1024/4096 cases cross tile boundaries
# end-to-end (partitioning math is COPY_BLOCK_SIZE-agnostic, so a small
# value proves it for all values).
_TILE_BLOCK_CONFIGS = [
(1, 1024),
(_TEMPORAL_TILES, 1024),
(_TEMPORAL_TILES, 8),
]
@_parametrize("copy_size", _COPY_SIZES)
@_parametrize("src_off,dst_off", _ALIGN_PAIRS)
@_parametrize("num_tiles,copy_block_size", _TILE_BLOCK_CONFIGS)
def test_memcpy_u64_tiled_matches_slice_copy(
copy_size, src_off, dst_off, num_tiles, copy_block_size
):
device = torch.device("cuda")
torch.manual_seed(0)
# Pad by the max possible offset so the reference slice is in-bounds.
slack = _MAX_ALIGN_OFF + 8
src = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
# Start dst from a distinct random pattern so the "unchanged region"
# check catches any accidental out-of-range writes.
dst = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
ref = dst.clone()
ref[dst_off : dst_off + copy_size] = src[src_off : src_off + copy_size]
_memcpy_wrapper_kernel[(num_tiles,)](
src,
dst,
copy_size,
src_off,
dst_off,
COPY_BLOCK_SIZE=copy_block_size,
NUM_TILES=num_tiles,
)
torch.accelerator.synchronize()
# Byte-identical: the memcpy is dtype-agnostic; any diff is a bug.
torch.testing.assert_close(dst, ref, rtol=0, atol=0)
if __name__ == "__main__":
for cs in _COPY_SIZES:
for so, do in _ALIGN_PAIRS:
for nt, cbs in _TILE_BLOCK_CONFIGS:
test_memcpy_u64_tiled_matches_slice_copy(cs, so, do, nt, cbs)
print("ok")
@@ -1,512 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
The fused "align" pre-copy must migrate mamba state across block boundaries
with byte-identical semantics to the scalar V1 copy specs
(``get_conv_copy_spec`` / ``get_temporal_copy_spec``):
* conv state (SD layout, conv_width > 0): shift the sliding window by
``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
``state[bt[dst_col], :conv_width - token_bias]``.
* temporal state (conv_width == 0): ``token_bias`` selects the accepted
speculative column -- ``state[bt[src_col + token_bias]]`` ->
``state[bt[dst_col]]``.
The kernel must also no-op when ``src_col < 0`` (fresh request) or
``src_col == dst_col`` (no boundary crossed). V2 callers pass an explicit
``idx_mapping``; V1 align preprocessing launches in batch order with
``idx_mapping=None``.
"""
from __future__ import annotations
from collections.abc import Callable
from types import SimpleNamespace
from typing import Any
import numpy as np
import torch
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
from vllm.platforms import current_platform
from vllm.v1.worker import mamba_utils as worker_mamba_utils
from vllm.v1.worker.mamba_utils import (
_TEMPORAL_TILES,
postprocess_mamba_fused_kernel,
precopy_mamba_align_fused_kernel,
)
_parametrize: Callable[..., Callable[[Any], Any]]
try:
import pytest
_cuda_required = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
)
_parametrize = pytest.mark.parametrize
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
def _cuda_required(fn):
return fn
def _no_parametrize(_name, _values):
def _deco(fn):
return fn
return _deco
_parametrize = _no_parametrize
NUM_LAYERS = 3
CONV_WIDTH = 4 # conv_kernel - 1 + num_spec
CONV_DIM = 96
SSM_SHAPE = (4, 16, 16)
MAX_COLS = 8
def _build_state(num_blocks, device, conv_state_dim_first):
"""Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
convs, ssms = [], []
for _ in range(NUM_LAYERS):
conv_shape = (
(num_blocks, CONV_DIM, CONV_WIDTH)
if conv_state_dim_first
else (num_blocks, CONV_WIDTH, CONV_DIM)
)
convs.append(
torch.randn(
*conv_shape,
dtype=torch.bfloat16,
device=device,
)
)
ssms.append(
torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
)
return convs, ssms
def _build_meta(convs, ssms, device, conv_state_dim_first):
"""Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
n = NUM_LAYERS * 2
base = torch.zeros(n, dtype=torch.int64, device=device)
blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
elem = torch.zeros(n, dtype=torch.int32, device=device)
inner = torch.zeros(n, dtype=torch.int64, device=device)
width = torch.zeros(n, dtype=torch.int32, device=device)
group = torch.zeros(n, dtype=torch.int32, device=device)
drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD)
drs = torch.zeros(n, dtype=torch.int64, device=device)
i = 0
for layer in range(NUM_LAYERS):
conv, ssm = convs[layer], ssms[layer]
# conv (SD): width = size(1), inner = stride(1)
base[i] = conv.data_ptr()
blk_stride[i] = conv.stride(0) * conv.element_size()
elem[i] = conv.element_size()
if conv_state_dim_first:
width[i] = conv.size(2)
inner[i] = 1
drc[i] = conv.size(1)
drs[i] = conv.stride(1) * conv.element_size()
else:
width[i] = conv.size(1)
inner[i] = conv.stride(1)
i += 1
# ssm (temporal): width = 0, inner = elems per block
base[i] = ssm.data_ptr()
blk_stride[i] = ssm.stride(0) * ssm.element_size()
elem[i] = ssm.element_size()
width[i] = 0
inner[i] = ssm[0].numel()
i += 1
return base, blk_stride, elem, inner, width, group, drc, drs
def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs, conv_dim_first):
"""Apply the V1 copy semantics on clones, reading from the pre-copy state."""
conv_pre = [c.clone() for c in convs]
ssm_pre = [s.clone() for s in ssms]
conv_ref = [c.clone() for c in convs]
ssm_ref = [s.clone() for s in ssms]
for r in range(num_reqs):
sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
if sc < 0 or sc == dc:
continue
sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias
for layer in range(NUM_LAYERS):
if conv_dim_first:
conv_ref[layer][dblk, :, : CONV_WIDTH - tb] = conv_pre[layer][
sblk, :, tb:
]
else:
conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
return conv_ref, ssm_ref
@_parametrize("conv_state_dim_first", [False, True])
@_parametrize("num_reqs", [1, 4, 16])
@_parametrize("token_bias", [0, 1, 2])
@_parametrize("has_idx_mapping", [True, False])
@_parametrize("temporal_tiles", [1, _TEMPORAL_TILES])
@_cuda_required
def test_precopy_matches_v1_copy_specs(
num_reqs,
token_bias,
has_idx_mapping,
conv_state_dim_first,
temporal_tiles,
):
device = torch.device("cuda")
torch.manual_seed(0)
# Distinct physical block per (req, col) so copies never alias.
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
for r in range(num_reqs):
bt[r] = torch.arange(
1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
)
# Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
# the rest cross from col 1 -> col 0 with the given spec token bias.
src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
if num_reqs >= 1:
src_col[0] = -1 # fresh -> no copy
if num_reqs >= 2:
dst_col[1] = 1 # src_col == dst_col -> no copy
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
conv_ref, ssm_ref = _reference(
convs,
ssms,
bt.cpu(),
src_col.cpu(),
dst_col.cpu(),
bias.cpu(),
num_reqs,
conv_state_dim_first,
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
# Non-identity mapping: block-table rows are request slots, not batch rows.
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device).flip(0)
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
precopy_mamba_align_fused_kernel[grid](
dst_col,
src_col,
bias,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
idx_mapping if has_idx_mapping else None,
num_reqs,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=conv_state_dim_first,
HAS_IDX_MAPPING=has_idx_mapping,
TEMPORAL_TILES=temporal_tiles,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
@_parametrize("conv_state_dim_first", [False, True])
@_cuda_required
def test_v2_postprocess_indexes_block_tables_by_req_slot(conv_state_dim_first):
"""Under PP the V2 postprocess runs after later steps have re-gathered the
batch-ordered tables, so it must index the request-slot tables by slot."""
device = torch.device("cuda")
torch.manual_seed(0)
num_reqs = 4
block_size = 8
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.arange(1, num_blocks, dtype=torch.int32, device=device).view(
num_reqs, MAX_COLS
)
# slot 0: copy col 1 -> 0; slot 1: src == dst; slot 2: not aligned;
# slot 3: copy col 2 -> 0 with one accepted draft token.
num_accepted = torch.tensor([1, 1, 1, 2], dtype=torch.int32, device=device)
state_idx = torch.tensor([1, 0, 1, 2], dtype=torch.int32, device=device)
new_num_computed = torch.tensor([8, 8, 5, 8], dtype=torch.int32, device=device)
ref_src = torch.tensor([1, 0, -1, 2], dtype=torch.int32)
ref_dst = torch.zeros(num_reqs, dtype=torch.int32)
ref_bias = torch.tensor([0, 0, 0, 1], dtype=torch.int32)
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
conv_ref, ssm_ref = _reference(
convs,
ssms,
bt.cpu(),
ref_src,
ref_dst,
ref_bias,
num_reqs,
conv_state_dim_first,
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device=device)
postprocess_mamba_fused_kernel[(num_reqs, NUM_LAYERS * 2, 1)](
num_accepted.clone(),
state_idx,
None,
new_num_computed,
None,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
num_accepted,
idx_mapping,
num_reqs,
block_size=block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=conv_state_dim_first,
HAS_IDX_MAPPING=True,
PRECOMPUTED_NEW_COMPUTED=True,
TEMPORAL_TILES=1,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
def test_ds_conv_copy_spec_reproduces_multi_accept_assert(monkeypatch):
monkeypatch.setattr(
layer_mamba_utils,
"is_conv_state_dim_first",
lambda: True,
)
state = torch.empty((2, CONV_DIM, CONV_WIDTH), dtype=torch.bfloat16)
with pytest.raises(AssertionError, match="num_accepted_tokens > 1"):
layer_mamba_utils.get_conv_copy_spec(
state=state,
block_ids=[0, 1],
cur_block_idx=0,
num_accepted_tokens=3,
)
class _FakeCpuGpuBuffer:
def __init__(self, n):
self.np = np.zeros(n, dtype=np.int32)
self.gpu = object()
self.copy_sizes = []
def copy_to_gpu(self, n=None):
self.copy_sizes.append(n)
return self.gpu
class _FakePrecopyContext:
def __init__(self, n):
self.is_initialized = True
self.mamba_group_ids = [0]
self.mamba_state_idx_buf = _FakeCpuGpuBuffer(n)
self.precopy_src_col_buf = _FakeCpuGpuBuffer(n)
self.precopy_token_bias_buf = _FakeCpuGpuBuffer(n)
self.calls = []
def initialize_from_forward_context(self, *args, **kwargs):
raise AssertionError("test context is pre-initialized")
def run_fused_precopy(
self,
*,
num_reqs,
state_idx_gpu,
src_col_gpu,
token_bias_gpu,
idx_mapping,
):
self.calls.append(
{
"num_reqs": num_reqs,
"state_idx": self.mamba_state_idx_buf.np[:num_reqs].copy(),
"src_col": self.precopy_src_col_buf.np[:num_reqs].copy(),
"token_bias": self.precopy_token_bias_buf.np[:num_reqs].copy(),
"idx_mapping": idx_mapping,
}
)
def _make_preprocess_case(token_bias):
req_ids = ["fresh", "same", "cross_a", "cross_b"]
scheduler_output = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_cached_reqs=SimpleNamespace(resumed_req_ids=set()),
num_scheduled_tokens={
"fresh": 1,
"same": 1,
"cross_a": 1,
"cross_b": 2,
},
)
input_batch = SimpleNamespace(
req_ids=req_ids,
num_accepted_tokens_cpu=np.array(
[token_bias + 1, token_bias + 1, token_bias + 1, 2],
dtype=np.int32,
),
)
requests = {
"fresh": SimpleNamespace(req_id="fresh", num_computed_tokens=0),
"same": SimpleNamespace(req_id="same", num_computed_tokens=5),
"cross_a": SimpleNamespace(req_id="cross_a", num_computed_tokens=8),
"cross_b": SimpleNamespace(req_id="cross_b", num_computed_tokens=7),
}
mamba_state_idx = {"same": 1, "cross_a": 0, "cross_b": 1}
return scheduler_output, input_batch, requests, mamba_state_idx
@_parametrize("token_bias", [1, 2])
def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bias):
block_size = 4
mamba_spec = SimpleNamespace(block_size=block_size, num_speculative_blocks=1)
# Fakes stand in for real config/context objects. The fused path tests
# below only touch the attributes preprocess_mamba actually reads.
cache_config: Any = SimpleNamespace(enable_prefix_caching=True)
kv_cache_config: Any = SimpleNamespace()
scalar_copy_calls = []
def fake_collect(
copy_bufs,
kv_cache_config,
mamba_state_copy_funcs,
mamba_group_ids,
src_block_idx,
dest_block_idx,
accept_token_bias,
req_state,
forward_context,
):
scalar_copy_calls.append(
(
req_state.req_id,
src_block_idx,
dest_block_idx,
accept_token_bias,
)
)
monkeypatch.setattr(worker_mamba_utils, "collect_mamba_copy_meta", fake_collect)
monkeypatch.setattr(
worker_mamba_utils, "do_mamba_copy_block", lambda copy_bufs: None
)
scalar_case = _make_preprocess_case(token_bias)
fused_case = _make_preprocess_case(token_bias)
scalar_copy_bufs: Any = SimpleNamespace(
mamba_group_ids=[0],
mamba_spec=mamba_spec,
offset=0,
)
worker_mamba_utils.preprocess_mamba(
scheduler_output=scalar_case[0],
kv_cache_config=kv_cache_config,
cache_config=cache_config,
mamba_state_idx=scalar_case[3],
input_batch=scalar_case[1],
requests=scalar_case[2],
forward_context={},
mamba_state_copy_funcs={},
copy_bufs=scalar_copy_bufs,
)
ctx: Any = _FakePrecopyContext(len(fused_case[1].req_ids))
fused_copy_bufs: Any = SimpleNamespace(
mamba_group_ids=[0],
mamba_spec=mamba_spec,
offset=0,
)
worker_mamba_utils.preprocess_mamba(
scheduler_output=fused_case[0],
kv_cache_config=kv_cache_config,
cache_config=cache_config,
mamba_state_idx=fused_case[3],
input_batch=fused_case[1],
requests=fused_case[2],
forward_context={},
mamba_state_copy_funcs={},
copy_bufs=fused_copy_bufs,
align_ctx=ctx,
)
assert fused_case[3] == scalar_case[3]
np.testing.assert_array_equal(
fused_case[1].num_accepted_tokens_cpu,
scalar_case[1].num_accepted_tokens_cpu,
)
assert scalar_copy_calls == [
("cross_a", 0, 2, token_bias),
("cross_b", 1, 2, 1),
]
assert len(ctx.calls) == 1
call = ctx.calls[0]
assert call["num_reqs"] == len(fused_case[1].req_ids)
assert call["idx_mapping"] is None
np.testing.assert_array_equal(call["state_idx"], np.array([0, 1, 2, 2]))
np.testing.assert_array_equal(call["src_col"], np.array([-1, -1, 0, 1]))
np.testing.assert_array_equal(call["token_bias"], np.array([0, 0, token_bias, 1]))
fused_copy_calls = [
(req_id, int(src), int(dst), int(bias))
for req_id, src, dst, bias in zip(
fused_case[1].req_ids,
call["src_col"],
call["state_idx"],
call["token_bias"],
)
if int(src) != -1 and int(src) != int(dst)
]
assert fused_copy_calls == scalar_copy_calls
if __name__ == "__main__":
from itertools import product
_CASES = product(
(1, 4, 16), # num_reqs
(0, 1, 2), # token_bias
(True, False), # has_idx_mapping
(False, True), # conv_state_dim_first
(1, _TEMPORAL_TILES), # temporal_tiles
)
for nr, tb, mapping, dim_first, tt in _CASES:
test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first, tt)
print(
f"OK num_reqs={nr} token_bias={tb} has_idx_mapping={mapping} "
f"conv_dim_first={dim_first} temporal_tiles={tt}"
)
@@ -1,248 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prefill == decode equivalence for the Mamba2 ReplaySSM kernels.
The SSM recurrence is path-independent: vLLM's chunked prefill (the SSD kernel
``mamba_chunk_scan_combined_varlen``) and step-by-step decode must produce the
same per-position outputs and final state over a sequence. This file feeds one
set of inputs through the production dt flow (raw dt + softplus + a per-head
dt_bias, applied inside each kernel) to:
* the exact fp32 step recurrence (``selective_state_update_ref``) -- the
ground truth,
* the chunked prefill kernel,
* the baseline decode kernel,
* the ReplaySSM output_only decode kernel,
and checks all of them agree. Prefill (a chunked scan) and decode (a step
recurrence) are different code paths, so they differ numerically: the chunked
scan carries ~2e-2 (fp32) / ~4e-2 (bf16) vs the exact recurrence, far above the
near-exact decode. We therefore anchor every path on the exact recurrence at
SSD-level tolerances (the same regime as ``test_mamba_ssm_ssd.py``), which the
chunked scan sets, keyed off the activation dtype.
State and activation/buffer precision are swept independently, including the
fp32-state + bf16-activation production config.
"""
import pytest
import torch
from tests.kernels.mamba.utils import selective_state_update_ref
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
selective_state_update_replayssm_output_only,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
def _prefill_tolerances(act_dtype: torch.dtype) -> tuple[float, float]:
# The chunked prefill scan, not the decode, sets these: it carries ~2e-2
# (fp32) / ~6e-2 (bf16) vs the exact recurrence, while ReplaySSM decode is
# near-exact (~2e-6 fp32). Keyed off the activation dtype (the outputs are
# in act_dtype). Same regime as test_mamba_ssm_ssd.py.
if act_dtype == torch.float32:
return 1e-2, 3e-2
return 6e-2, 1e-1
def _run_prefill_decode_equivalence(
*,
state_dtype: torch.dtype,
act_dtype: torch.dtype,
nheads: int,
headdim: int,
ngroups: int,
dstate: int,
seqlen: int,
chunk_size: int,
max_cache_len: int,
seed: int = 0,
) -> None:
"""Prefill the whole sequence and decode it step by step; check both match
the exact fp32 recurrence (and each other). All paths use the production dt
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
device = current_platform.device_type
rtol, atol = _prefill_tolerances(act_dtype)
set_random_seed(seed)
# Production dt flow: raw dt + a per-head dt_bias (~-4 keeps softplus(dt +
# bias) small and well-conditioned). dt_bias is (nheads,) for prefill and
# (nheads, headdim) for the decode kernels/reference.
A = -torch.exp(torch.rand(nheads, device=device, dtype=act_dtype))
dt = torch.randn(seqlen, nheads, device=device, dtype=act_dtype)
dt_bias = torch.rand(nheads, device=device, dtype=act_dtype) - 4
dt_bias_hd = dt_bias.view(nheads, 1).expand(nheads, headdim)
X = torch.randn(seqlen, nheads, headdim, device=device, dtype=act_dtype)
B = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
C = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
A_bcast = A.view(nheads, 1, 1).expand(nheads, headdim, dstate)
# Chunked prefill over the whole sequence (implicit batch=1, varlen). The
# kernel always returns the final state in fp32, so no state_dtype plumbing.
cu_seqlens = torch.tensor((0, seqlen), device=device).cumsum(0).to(torch.int32)
cu_chunk_seqlens, last_chunk_indices, seq_idx = compute_varlen_chunk_metadata(
cu_seqlens, chunk_size
)
y_prefill = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
final_state_prefill = mamba_chunk_scan_combined_varlen(
X,
dt,
A,
B,
C,
chunk_size,
cu_seqlens=cu_seqlens,
cu_chunk_seqlens=cu_chunk_seqlens,
last_chunk_indices=last_chunk_indices,
seq_idx=seq_idx,
out=y_prefill,
D=None,
dt_bias=dt_bias,
dt_softplus=True,
)
# Step paths: exact fp32 recurrence (ground truth), baseline, ReplaySSM.
# State follows state_dtype; caches follow act_dtype (dt_cache is fp32).
state_ref = torch.zeros(
1, nheads, headdim, dstate, device=device, dtype=torch.float32
)
state_base = torch.zeros(
1, nheads, headdim, dstate, device=device, dtype=state_dtype
)
state_dec = torch.zeros(
1, nheads, headdim, dstate, device=device, dtype=state_dtype
)
x_cache = torch.zeros(
1, nheads, max_cache_len, headdim, device=device, dtype=act_dtype
)
dt_cache = torch.zeros(1, nheads, max_cache_len, device=device, dtype=torch.float32)
B_cache = torch.zeros(
1, ngroups, max_cache_len, dstate, device=device, dtype=act_dtype
)
bc_pre = torch.empty(1, ngroups, max_cache_len, device=device, dtype=torch.float32)
write_pos = torch.zeros(1, dtype=torch.int32, device=device)
# No skip connection (D=0) on any path here; the D!=0 path is covered by the
# standard-decode suite. The baseline kernel needs a D tensor, not None.
D_zero = torch.zeros(nheads, headdim, device=device)
y_ref = torch.empty(seqlen, nheads, headdim, device=device, dtype=torch.float32)
y_base = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
y_dec = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
for t in range(seqlen):
dt_t = dt[t].view(1, nheads, 1).expand(1, nheads, headdim)
is_flush = write_pos == max_cache_len - 1
y_ref[t] = selective_state_update_ref(
state_ref,
X[t : t + 1].float(),
dt_t.float(),
A_bcast.float(),
B[t : t + 1].float(),
C[t : t + 1].float(),
dt_bias=dt_bias_hd.float(),
dt_softplus=True,
)[0]
out_b = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
selective_state_update(
state_base,
X[t : t + 1],
dt_t,
A_bcast,
B[t : t + 1],
C[t : t + 1],
D=D_zero,
dt_bias=dt_bias_hd,
dt_softplus=True,
out=out_b,
)
y_base[t] = out_b[0]
out_d = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
common = dict(
dt_bias=dt_bias_hd,
dt_softplus=True,
x_cache=x_cache,
dt_cache=dt_cache,
B_cache=B_cache,
write_pos=write_pos,
is_flush=is_flush,
max_cache_len=max_cache_len,
out=out_d,
)
selective_state_update_replayssm_output_only(
state_dec,
X[t : t + 1],
dt_t,
A_bcast,
B[t : t + 1],
C[t : t + 1],
bc_pre=bc_pre,
**common,
)
y_dec[t] = out_d[0]
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
# Every path computes the same recurrence; anchor each on the fp32 truth.
torch.testing.assert_close(y_prefill.float(), y_ref, rtol=rtol, atol=atol)
torch.testing.assert_close(y_base.float(), y_ref, rtol=rtol, atol=atol)
torch.testing.assert_close(y_dec.float(), y_ref, rtol=rtol, atol=atol)
# Headline: ReplaySSM decode matches the chunked prefill directly.
torch.testing.assert_close(y_dec.float(), y_prefill.float(), rtol=rtol, atol=atol)
# Final state too (the recurrence ends in state_ref after the loop).
torch.testing.assert_close(
final_state_prefill[0].float(), state_ref[0], rtol=rtol, atol=atol
)
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.parametrize(
"precision",
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
# appears as an activation dtype (s32_afp16, sfp16_afp16) and as a finer-
# mantissa state under bf16 activations (sfp16_a16).
[
pytest.param((torch.float32, torch.float32), id="s32_a32"),
pytest.param((torch.float32, torch.bfloat16), id="s32_a16"),
pytest.param((torch.bfloat16, torch.bfloat16), id="s16_a16"),
pytest.param((torch.float32, torch.float16), id="s32_afp16"),
pytest.param((torch.float16, torch.float16), id="sfp16_afp16"),
pytest.param((torch.float16, torch.bfloat16), id="sfp16_a16"),
],
)
@pytest.mark.parametrize(
"geometry", # (nheads, headdim, dstate, ngroups)
[
pytest.param((8, 64, 64, 2), id="small"),
pytest.param((96, 80, 128, 8), id="nano4b"),
],
)
def test_replayssm_prefill_decode_equivalence(
precision: tuple[torch.dtype, torch.dtype],
geometry: tuple[int, int, int, int],
):
state_dtype, act_dtype = precision
nheads, headdim, dstate, ngroups = geometry
_run_prefill_decode_equivalence(
state_dtype=state_dtype,
act_dtype=act_dtype,
nheads=nheads,
headdim=headdim,
ngroups=ngroups,
dstate=dstate,
seqlen=16,
chunk_size=8,
max_cache_len=4,
)