[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,472 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda_alike()
|
||||
or current_platform.is_xpu()
|
||||
or current_platform.is_cpu()
|
||||
),
|
||||
reason="causal_conv1d Triton kernels require CUDA-alike, XPU, or CPU",
|
||||
)
|
||||
|
||||
|
||||
def causal_conv1d_ref(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
initial_states: torch.Tensor | None = None,
|
||||
return_final_states: bool = False,
|
||||
final_states_out: torch.Tensor | None = None,
|
||||
activation: str | None = "silu",
|
||||
):
|
||||
"""x: (batch, dim, seqlen)
|
||||
weight: (dim, width)
|
||||
bias: (dim,)
|
||||
initial_states: (batch, dim, width - 1)
|
||||
final_states_out: (batch, dim, width - 1)
|
||||
|
||||
out: (batch, dim, seqlen)
|
||||
"""
|
||||
if activation not in [None, "silu", "swish"]:
|
||||
raise NotImplementedError("activation must be None, silu, or swish")
|
||||
dtype_in = x.dtype
|
||||
x = x.to(weight.dtype)
|
||||
seqlen = x.shape[-1]
|
||||
dim, width = weight.shape
|
||||
if initial_states is None:
|
||||
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim)
|
||||
else:
|
||||
x = torch.cat([initial_states, x], dim=-1)
|
||||
out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim)
|
||||
out = out[..., :seqlen]
|
||||
if return_final_states:
|
||||
final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to(
|
||||
dtype_in
|
||||
) # (batch, dim, width - 1)
|
||||
if final_states_out is not None:
|
||||
final_states_out.copy_(final_states)
|
||||
else:
|
||||
final_states_out = final_states
|
||||
out = (out if activation is None else F.silu(out)).to(dtype=dtype_in)
|
||||
return (out, None) if not return_final_states else (out, final_states_out)
|
||||
|
||||
|
||||
def causal_conv1d_update_ref(
|
||||
x, conv_state, weight, bias=None, activation=None, cache_seqlens=None
|
||||
):
|
||||
"""x: (batch, dim) or (batch, dim, seqlen)
|
||||
conv_state: (batch, dim, state_len), where state_len >= width - 1
|
||||
weight: (dim, width)
|
||||
bias: (dim,)
|
||||
cache_seqlens: (batch,), dtype int32.
|
||||
If not None, the conv_state is treated as a circular buffer.
|
||||
The conv_state will be updated by copying x to the
|
||||
conv_state starting at the index
|
||||
@cache_seqlens % state_len before performing the convolution.
|
||||
|
||||
out: (batch, dim) or (batch, dim, seqlen)
|
||||
"""
|
||||
if activation not in [None, "silu", "swish"]:
|
||||
raise NotImplementedError("activation must be None, silu, or swish")
|
||||
dtype_in = x.dtype
|
||||
unsqueeze = x.dim() == 2
|
||||
if unsqueeze:
|
||||
x = x.unsqueeze(-1)
|
||||
batch, dim, seqlen = x.shape
|
||||
width = weight.shape[1]
|
||||
state_len = conv_state.shape[-1]
|
||||
assert conv_state.shape == (batch, dim, state_len)
|
||||
assert weight.shape == (dim, width)
|
||||
if cache_seqlens is None:
|
||||
x_new = torch.cat([conv_state, x], dim=-1).to(
|
||||
weight.dtype
|
||||
) # (batch, dim, state_len + seqlen)
|
||||
conv_state.copy_(x_new[:, :, -state_len:])
|
||||
else:
|
||||
width_idx = torch.arange(
|
||||
-(width - 1), 0, dtype=torch.long, device=x.device
|
||||
).unsqueeze(0) + cache_seqlens.unsqueeze(1)
|
||||
width_idx = (
|
||||
torch.remainder(width_idx, state_len).unsqueeze(1).expand(-1, dim, -1)
|
||||
)
|
||||
x_new = torch.cat([conv_state.gather(2, width_idx), x], dim=-1).to(weight.dtype)
|
||||
copy_idx = torch.arange(seqlen, dtype=torch.long, device=x.device).unsqueeze(
|
||||
0
|
||||
) + cache_seqlens.unsqueeze(1)
|
||||
copy_idx = torch.remainder(copy_idx, state_len).unsqueeze(1).expand(-1, dim, -1)
|
||||
conv_state.scatter_(2, copy_idx, x)
|
||||
out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0, groups=dim)[
|
||||
:, :, -seqlen:
|
||||
]
|
||||
if unsqueeze:
|
||||
out = out.squeeze(-1)
|
||||
return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [False, True])
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@pytest.mark.parametrize("seqlen", [1])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
|
||||
def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype):
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
# set seed
|
||||
set_random_seed(0)
|
||||
batch = 2
|
||||
x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
|
||||
x_ref = x.clone()
|
||||
# +1 entry to reserve index 0 as null block
|
||||
conv_state = torch.randn(batch + 1, dim, width - 1, device=device, dtype=itype)
|
||||
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
# Start indices from 1, skipping null block at index 0
|
||||
conv_state_indices = torch.arange(1, batch + 1, dtype=torch.int32, device=device)
|
||||
conv_state_ref = conv_state[conv_state_indices].detach().clone()
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
out = causal_conv1d_update(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
)
|
||||
out_ref = causal_conv1d_update_ref(
|
||||
x_ref, conv_state_ref, weight, bias, activation=activation
|
||||
)
|
||||
|
||||
assert torch.equal(conv_state[conv_state_indices], conv_state_ref)
|
||||
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [False, True])
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@pytest.mark.parametrize("seqlen", [1, 3])
|
||||
@pytest.mark.parametrize("width", [3, 4])
|
||||
@pytest.mark.parametrize("dim", [2048 + 16, 4096])
|
||||
# tests correctness in case subset of the sequences are padded
|
||||
@pytest.mark.parametrize("with_padding", [True, False])
|
||||
@pytest.mark.parametrize("batch_size", [3])
|
||||
def test_causal_conv1d_update_with_batch_gather(
|
||||
batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
|
||||
# set seed
|
||||
set_random_seed(0)
|
||||
|
||||
padding = 5 if with_padding else 0
|
||||
padded_batch_size = batch_size + padding
|
||||
# total_entries = number of cache line
|
||||
total_entries = 10 * batch_size
|
||||
|
||||
# x will be (batch, dim, seqlen) with contiguous along dim-axis
|
||||
x = torch.randn(
|
||||
padded_batch_size, seqlen, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
|
||||
x_ref = x.clone()
|
||||
|
||||
# +1 to exclude index 0 (null block)
|
||||
conv_state_indices = (torch.randperm(total_entries - 1)[:batch_size] + 1).to(
|
||||
dtype=torch.int32, device=device
|
||||
)
|
||||
unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device)
|
||||
unused_states_bool[conv_state_indices] = False
|
||||
padded_state_indices = torch.concat(
|
||||
[
|
||||
conv_state_indices,
|
||||
torch.as_tensor(
|
||||
[NULL_BLOCK_ID] * padding, dtype=torch.int32, device=device
|
||||
),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
# conv_state will be (cache_lines, dim, state_len)
|
||||
# with contiguous along dim-axis
|
||||
conv_state = torch.randn(
|
||||
total_entries, width - 1, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
|
||||
conv_state_for_padding_test = conv_state.clone()
|
||||
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
conv_state_ref = conv_state[conv_state_indices, :].detach().clone()
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
out = causal_conv1d_update(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=padded_state_indices,
|
||||
)
|
||||
out_ref = causal_conv1d_update_ref(
|
||||
x_ref[:batch_size], conv_state_ref, weight, bias, activation=activation
|
||||
)
|
||||
|
||||
assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
|
||||
assert torch.equal(
|
||||
conv_state[unused_states_bool], conv_state_for_padding_test[unused_states_bool]
|
||||
)
|
||||
assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [True])
|
||||
@pytest.mark.parametrize("has_bias", [True])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("seqlen", [8, 249, 4096])
|
||||
@pytest.mark.parametrize("dim", [64, 4096])
|
||||
@pytest.mark.parametrize("with_padding", [True, False])
|
||||
@pytest.mark.parametrize("batch", [4, 10])
|
||||
def test_causal_conv1d_varlen(
|
||||
batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
if not current_platform.is_cpu():
|
||||
torch.accelerator.empty_cache()
|
||||
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
|
||||
if itype == torch.bfloat16:
|
||||
rtol, atol = 1e-2, 5e-2
|
||||
# set seed
|
||||
set_random_seed(0)
|
||||
seqlens = []
|
||||
batch_size = batch
|
||||
padding = 3 if with_padding else 0
|
||||
padded_batch_size = batch_size + padding
|
||||
nsplits = padded_batch_size - 1
|
||||
|
||||
eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
|
||||
|
||||
seqlens.append(
|
||||
torch.diff(
|
||||
torch.cat([torch.tensor([-1]), eos_pos, torch.tensor([seqlen - 1])])
|
||||
).tolist()
|
||||
)
|
||||
assert sum(seqlens[-1]) == seqlen
|
||||
assert all(s > 0 for s in seqlens[-1])
|
||||
|
||||
total_entries = batch_size * 10
|
||||
cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
|
||||
cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum], dim=0)
|
||||
x = rearrange(
|
||||
torch.randn(1, seqlen, 4096 + dim + 64, device=device, dtype=itype),
|
||||
"b s d -> b d s",
|
||||
)[:, 4096 : 4096 + dim, :]
|
||||
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
x_ref = x.clone()
|
||||
weight_ref = weight.clone()
|
||||
bias_ref = bias.clone() if bias is not None else None
|
||||
activation = None if not silu_activation else "silu"
|
||||
final_states = torch.randn(
|
||||
total_entries, width - 1, dim, device=x.device, dtype=x.dtype
|
||||
).transpose(1, 2)
|
||||
final_states_ref = final_states.clone()
|
||||
has_initial_states = torch.randint(
|
||||
0, 2, (cumsum.shape[0] - 1,), dtype=torch.bool, device=x.device
|
||||
)
|
||||
# +1 to exclude index 0 (null block)
|
||||
state_indices = (
|
||||
torch.randperm(total_entries - 1, dtype=torch.int32, device=x.device)[
|
||||
:batch_size
|
||||
]
|
||||
+ 1
|
||||
)
|
||||
padded_state_indices = torch.concat(
|
||||
[
|
||||
state_indices,
|
||||
torch.as_tensor(
|
||||
[NULL_BLOCK_ID] * padding, dtype=torch.int32, device=device
|
||||
),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
out = causal_conv1d_fn(
|
||||
x.squeeze(0),
|
||||
weight,
|
||||
bias=bias,
|
||||
conv_states=final_states,
|
||||
query_start_loc=cumsum.to(device),
|
||||
cache_indices=padded_state_indices,
|
||||
has_initial_state=has_initial_states,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
out_ref = []
|
||||
out_ref_b = []
|
||||
|
||||
splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)]
|
||||
for i in range(len(seqlens[0])):
|
||||
x_s = [v[i].unsqueeze(0) for v in splits][0]
|
||||
if padded_state_indices[i] == NULL_BLOCK_ID:
|
||||
continue
|
||||
out_ref_b.append(
|
||||
causal_conv1d_ref(
|
||||
x_s,
|
||||
weight_ref,
|
||||
bias_ref,
|
||||
activation=activation,
|
||||
return_final_states=True,
|
||||
final_states_out=final_states_ref[padded_state_indices[i]].unsqueeze(0),
|
||||
initial_states=final_states_ref[padded_state_indices[i]].unsqueeze(0)
|
||||
if has_initial_states[i]
|
||||
else None,
|
||||
)
|
||||
)
|
||||
out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2))
|
||||
out_ref_tensor = torch.cat(out_ref, dim=0)
|
||||
|
||||
assert torch.allclose(
|
||||
final_states[state_indices],
|
||||
final_states_ref[state_indices],
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
|
||||
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [True])
|
||||
@pytest.mark.parametrize("has_bias", [True])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [64, 2048])
|
||||
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
|
||||
)
|
||||
def test_causal_conv1d_update_ragged_varlen(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
mk_inputs = lambda: _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
)
|
||||
inputs, state = mk_inputs()
|
||||
actual = causal_conv1d_update(conv_state=state, **inputs)
|
||||
|
||||
ref_inputs, ref_state = mk_inputs()
|
||||
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
|
||||
|
||||
for i, ref_out in enumerate(ref_outs):
|
||||
start = int(inputs["query_start_loc"][i])
|
||||
end = int(inputs["query_start_loc"][i + 1])
|
||||
entry = inputs["conv_state_indices"][i : i + 1]
|
||||
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
|
||||
assert torch.equal(state[entry], ref_state[entry])
|
||||
|
||||
|
||||
def _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
set_random_seed(0)
|
||||
|
||||
batch_size = len(q_lens_cfg)
|
||||
max_query_len = 5
|
||||
assert max(q_lens_cfg) <= max_query_len
|
||||
state_len = width - 1 + max_query_len
|
||||
total_entries = 10 * batch_size
|
||||
|
||||
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
total_tokens = int(q_lens.sum())
|
||||
|
||||
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
|
||||
conv_state = torch.randn(
|
||||
total_entries, state_len, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
conv_state_indices = (
|
||||
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
|
||||
).to(torch.int32)
|
||||
num_accepted_tokens = torch.randint(
|
||||
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs = dict(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation=activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
return inputs, conv_state
|
||||
|
||||
|
||||
def _create_ragged_varlen_ref_outs(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation,
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
max_query_len,
|
||||
) -> Iterator[torch.Tensor]:
|
||||
for seq_idx in range(len(conv_state_indices)):
|
||||
start = int(query_start_loc[seq_idx])
|
||||
end = int(query_start_loc[seq_idx + 1])
|
||||
entry = conv_state_indices[seq_idx : seq_idx + 1]
|
||||
if start == end:
|
||||
yield x[start:end]
|
||||
continue
|
||||
out_ref = causal_conv1d_update(
|
||||
x[start:end],
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=entry,
|
||||
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
|
||||
query_start_loc=query_start_loc.new_tensor([0, end - start]),
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
yield out_ref
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.mamba.short_conv import ShortConv
|
||||
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_dist():
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.parallel_state.model_parallel_is_initialized",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.parallel_state.get_tp_group",
|
||||
return_value=MagicMock(rank_in_group=0),
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_config():
|
||||
# ShortConv only needs compilation_config from the current vLLM config, so a
|
||||
# minimal config (model_config=None) avoids mocking ModelConfig and the
|
||||
# associated VllmConfig validation churn.
|
||||
return VllmConfig(compilation_config=CompilationConfig())
|
||||
|
||||
|
||||
def test_short_conv_forward_native_prefill(vllm_config):
|
||||
prefix = "test_layer"
|
||||
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
|
||||
dim = 16
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
|
||||
|
||||
layer.to("cpu")
|
||||
# vLLM Linear layers allocate weights with torch.empty (uninitialized).
|
||||
# On ARM these come back as zero-filled pages, so in_proj output is zero and
|
||||
# the prefill state stays zero. Seed + init to make the test platform-safe.
|
||||
torch.manual_seed(0)
|
||||
for p in layer.parameters():
|
||||
torch.nn.init.normal_(p)
|
||||
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
|
||||
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
|
||||
|
||||
# Mock AttentionMetadata
|
||||
num_prefills = 1
|
||||
num_prefill_tokens = 5
|
||||
query_start_loc_p = torch.tensor([0, 5], dtype=torch.int32)
|
||||
state_indices_tensor_p = torch.tensor([0], dtype=torch.int32)
|
||||
|
||||
# ShortConvAttentionMetadata
|
||||
attn_metadata = ShortConvAttentionMetadata(
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_reqs=1,
|
||||
query_start_loc_p=query_start_loc_p,
|
||||
has_initial_states_p=torch.tensor([False]),
|
||||
state_indices_tensor_p=state_indices_tensor_p,
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
# Mock KV cache
|
||||
# conv_state shape (num_blocks, L_cache - 1, dim)
|
||||
conv_state = torch.zeros((1, config.conv_L_cache - 1, dim))
|
||||
layer.kv_cache = (conv_state,)
|
||||
|
||||
hidden_states = torch.randn((num_prefill_tokens, dim))
|
||||
output = torch.zeros_like(hidden_states)
|
||||
|
||||
attn_metadata_dict = {prefix: attn_metadata}
|
||||
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
|
||||
layer.forward_native(hidden_states, output)
|
||||
|
||||
# Check if KV cache was updated
|
||||
assert not torch.allclose(conv_state, torch.zeros_like(conv_state))
|
||||
|
||||
|
||||
def test_short_conv_forward_native_decode(vllm_config):
|
||||
prefix = "test_layer_decode"
|
||||
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
|
||||
dim = 16
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
|
||||
|
||||
layer.to("cpu")
|
||||
torch.manual_seed(0)
|
||||
for p in layer.parameters():
|
||||
torch.nn.init.normal_(p)
|
||||
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
|
||||
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
|
||||
|
||||
# Mock AttentionMetadata for 2 decode requests
|
||||
num_decodes = 2
|
||||
state_indices_tensor_d = torch.tensor([0, 1], dtype=torch.int32)
|
||||
|
||||
attn_metadata = ShortConvAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes,
|
||||
num_reqs=num_decodes,
|
||||
query_start_loc_p=None,
|
||||
has_initial_states_p=None,
|
||||
state_indices_tensor_p=torch.empty((0,), dtype=torch.int32),
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
# Mock KV cache (2 blocks for 2 requests)
|
||||
conv_state = torch.randn((2, config.conv_L_cache - 1, dim))
|
||||
layer.kv_cache = (conv_state,)
|
||||
|
||||
hidden_states = torch.randn((num_decodes, dim))
|
||||
output = torch.zeros_like(hidden_states)
|
||||
|
||||
old_conv_state = conv_state.clone()
|
||||
|
||||
attn_metadata_dict = {prefix: attn_metadata}
|
||||
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
|
||||
layer.forward_native(hidden_states, output)
|
||||
|
||||
# Check if KV cache was updated
|
||||
assert not torch.allclose(conv_state, old_conv_state)
|
||||
|
||||
|
||||
def test_dispatch_cpu_unquantized_gemm_conv_layer():
|
||||
# Convolution layers have >2D weights; dispatch should skip them gracefully.
|
||||
# Shape/dtype are AMX-pack safe (bf16, width==4, dim % block_size == 0) so
|
||||
# the AMX prepack branch does not raise on AMX-capable CPUs.
|
||||
class MockConvLayer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(
|
||||
torch.randn(32, 1, 4, dtype=torch.bfloat16)
|
||||
)
|
||||
self.bias = torch.nn.Parameter(torch.randn(32, dtype=torch.bfloat16))
|
||||
|
||||
layer = MockConvLayer()
|
||||
# The ndim != 2 guard returns early without raising.
|
||||
dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
# No cpu_linear set — conv layers are handled elsewhere.
|
||||
assert not hasattr(layer, "cpu_linear")
|
||||
@@ -1,392 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-path tests for fused Qwen3.5 GDN MTP decode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (current_platform.is_cuda() and current_platform.has_device_capability(80)):
|
||||
pytest.skip(
|
||||
reason="Fused GDN MTP decode requires CUDA compute capability 8.0+.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from tests.v1.attention.utils import ( # noqa: E402
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig, set_current_vllm_config # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
|
||||
ChunkGatedDeltaRule,
|
||||
QwenGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noqa: E402
|
||||
rmsnorm_fn,
|
||||
)
|
||||
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
|
||||
|
||||
NUM_SPEC = 3
|
||||
SPEC_TOKENS = NUM_SPEC + 1
|
||||
H = 1
|
||||
HV = 8
|
||||
K = 128
|
||||
V = 128
|
||||
CONV_KERNEL = 4
|
||||
CONV_DIM = 2 * H * K + HV * V
|
||||
BLOCK_SIZE = 16
|
||||
PREFIX = "model.layers.0.linear_attn"
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor, activation: str) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = activation
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
x,
|
||||
self.weight,
|
||||
None,
|
||||
z=z,
|
||||
eps=self.eps,
|
||||
norm_before_gate=True,
|
||||
activation=self.activation,
|
||||
)
|
||||
|
||||
|
||||
def _make_vllm_config():
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
hf_config_override={"linear_key_head_dim": K},
|
||||
)
|
||||
config.additional_config = {"gdn_prefill_backend": "cutedsl"}
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=NUM_SPEC
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _build_layer(
|
||||
vllm_config,
|
||||
conv_state: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
a_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
output_gate_activation: str,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_packed_recurrent_decode=False,
|
||||
disable_tp_for_ba_proj=False,
|
||||
tp_size=1,
|
||||
num_k_heads=H,
|
||||
num_v_heads=HV,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
key_dim=K,
|
||||
value_dim=HV * V,
|
||||
activation="silu",
|
||||
A_log=a_log,
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight, output_gate_activation),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
for name in (
|
||||
"rearrange_mixed_qkv",
|
||||
"_forward_core",
|
||||
"_forward_core_decode_spec_post_conv_fused_norm",
|
||||
"_forward_core_decode_spec_fused_norm",
|
||||
"_can_use_fused_gdn_mtp_decode",
|
||||
"_rms_norm_gated_cuda",
|
||||
"_forward_core_fused_norm",
|
||||
"_forward_core_fused_norm_packed",
|
||||
"split_ba",
|
||||
):
|
||||
setattr(
|
||||
layer,
|
||||
name,
|
||||
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_v_heads,expected",
|
||||
[
|
||||
pytest.param(2, True, id="ratio1"),
|
||||
pytest.param(4, True, id="ratio2"),
|
||||
pytest.param(6, True, id="ratio3"),
|
||||
pytest.param(8, True, id="ratio4"),
|
||||
pytest.param(16, True, id="ratio8"),
|
||||
pytest.param(5, False, id="non-integral-ratio"),
|
||||
pytest.param(10, False, id="unsupported-ratio5"),
|
||||
],
|
||||
)
|
||||
def test_fused_mtp_head_ratio_guard(num_v_heads: int, expected: bool) -> None:
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
pytest.skip("fused GDN decode MTP op is not built")
|
||||
|
||||
layer = types.SimpleNamespace(
|
||||
num_k_heads=2,
|
||||
num_v_heads=num_v_heads,
|
||||
kv_cache=(None, torch.empty(1, dtype=torch.float32, device="cuda")),
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
attn_metadata = types.SimpleNamespace(
|
||||
spec_state_indices_tensor=torch.ones(
|
||||
1, SPEC_TOKENS, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
spec_sequence_masks=object(),
|
||||
num_decodes=0,
|
||||
num_spec_decodes=1,
|
||||
)
|
||||
|
||||
assert (
|
||||
QwenGatedDeltaNetAttention._can_use_fused_gdn_mtp_decode(
|
||||
cast(QwenGatedDeltaNetAttention, layer),
|
||||
cast(GDNAttentionMetadata, attn_metadata),
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 3
|
||||
hidden_states = torch.empty(num_tokens, 1, dtype=torch.bfloat16, device=device)
|
||||
mixed_qkvz = torch.randn(
|
||||
num_tokens, CONV_DIM + HV * V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_fused_gdn_spec_decode=True,
|
||||
norm=types.SimpleNamespace(
|
||||
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
|
||||
),
|
||||
num_v_heads=HV,
|
||||
tp_size=1,
|
||||
head_v_dim=V,
|
||||
in_proj_qkvz=lambda _: (mixed_qkvz, None),
|
||||
in_proj_ba=lambda _: (ba, None),
|
||||
out_proj=lambda x: (x, None),
|
||||
)
|
||||
layer.forward_cuda = types.MethodType(
|
||||
QwenGatedDeltaNetAttention.forward_cuda, layer
|
||||
)
|
||||
|
||||
def packed_op(
|
||||
actual_qkvz: torch.Tensor,
|
||||
actual_ba: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
*,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
assert actual_qkvz is mixed_qkvz
|
||||
assert actual_ba is ba
|
||||
assert layer_name == _encode_layer_name(PREFIX)
|
||||
output.fill_(1)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
torch.ops.vllm,
|
||||
"qwen_gdn_attention_core_fused_norm_packed",
|
||||
side_effect=packed_op,
|
||||
) as packed_mock,
|
||||
patch.object(torch.ops.vllm, "qwen_gdn_attention_core") as triton_mock,
|
||||
):
|
||||
output = layer.forward_cuda(hidden_states)
|
||||
|
||||
packed_mock.assert_called_once()
|
||||
triton_mock.assert_not_called()
|
||||
torch.testing.assert_close(output, torch.ones_like(output))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,query_lens,draft_tokens,expected_fused_calls",
|
||||
[
|
||||
pytest.param([128], [SPEC_TOKENS], [NUM_SPEC], 1, id="pure-mtp"),
|
||||
pytest.param(
|
||||
[128, 96],
|
||||
[SPEC_TOKENS, 64],
|
||||
[NUM_SPEC, -1],
|
||||
0,
|
||||
id="mixed-mtp-falls-back",
|
||||
),
|
||||
pytest.param([96], [64], [-1], 0, id="pure-prefill"),
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
device = torch.device("cuda")
|
||||
vllm_config = _make_vllm_config()
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
common.block_table_tensor.add_(1)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
num_accepted_tokens=torch.ones(
|
||||
batch.batch_size, dtype=torch.int32, device=device
|
||||
),
|
||||
num_decode_draft_tokens_cpu=torch.tensor(draft_tokens, dtype=torch.int32),
|
||||
)
|
||||
|
||||
state_indices = [
|
||||
indices
|
||||
for indices in (
|
||||
metadata.spec_state_indices_tensor,
|
||||
metadata.non_spec_state_indices_tensor,
|
||||
)
|
||||
if indices is not None and indices.numel() > 0
|
||||
]
|
||||
pool_size = max(int(indices.max().item()) for indices in state_indices) + 1
|
||||
conv_state_shape, temporal_state_shape = (
|
||||
MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
1, H, HV, K, V, CONV_KERNEL, NUM_SPEC
|
||||
)
|
||||
)
|
||||
conv_state_seed = 0.05 * torch.randn(
|
||||
pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
ssm_state_seed = 0.01 * torch.randn(
|
||||
pool_size, *temporal_state_shape, dtype=torch.float32, device=device
|
||||
)
|
||||
a_log = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
conv_weight = 0.1 * torch.randn(
|
||||
CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
norm_weight = torch.randn(V, dtype=torch.float32, device=device)
|
||||
num_tokens = batch.compute_num_tokens()
|
||||
mixed_qkv = 0.1 * torch.randn(
|
||||
num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
b = 0.1 * torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device)
|
||||
a = 0.1 * torch.randn_like(b)
|
||||
output_gate = 0.1 * torch.randn(
|
||||
num_tokens, HV, V, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
mixed_qkvz = torch.cat((mixed_qkv, output_gate.flatten(1)), dim=-1)
|
||||
ba = torch.cat((b, a), dim=-1)
|
||||
context = types.SimpleNamespace(attn_metadata={PREFIX: metadata})
|
||||
|
||||
reference_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
qwen_gdn_linear_attn, "get_forward_context", return_value=context
|
||||
):
|
||||
reference_layer._forward_core(
|
||||
mixed_qkv=mixed_qkv.clone(),
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=reference_out,
|
||||
)
|
||||
reference_out = reference_layer.norm(reference_out, output_gate)
|
||||
|
||||
fused_layer = _build_layer(
|
||||
vllm_config,
|
||||
conv_state_seed.clone(),
|
||||
ssm_state_seed.clone(),
|
||||
a_log,
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
fused_op = qwen_gdn_linear_attn.ops.fused_gdn_decode_post_conv_mtp
|
||||
with (
|
||||
patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=context),
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.ops,
|
||||
"fused_gdn_decode_post_conv_mtp",
|
||||
wraps=fused_op,
|
||||
) as fused_mock,
|
||||
):
|
||||
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
|
||||
mixed_qkvz.clone(),
|
||||
ba,
|
||||
fused_out,
|
||||
layer_name=_encode_layer_name(PREFIX),
|
||||
)
|
||||
|
||||
assert fused_mock.call_count == expected_fused_calls
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[0], reference_layer.kv_cache[0], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(fused_out, reference_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(
|
||||
fused_layer.kv_cache[1],
|
||||
reference_layer.kv_cache[1],
|
||||
atol=3e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
reason="FlashInfer GDN prefill is not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import flashinfer.gdn_prefill # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
fi_chunk_gated_delta_rule,
|
||||
) # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
plan_flashinfer_checkpoints,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
|
||||
|
||||
|
||||
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
captured_cu_seqlens = None
|
||||
|
||||
def fake_chunk_gated_delta_rule(**kwargs):
|
||||
nonlocal captured_cu_seqlens
|
||||
captured_cu_seqlens = kwargs["cu_seqlens"]
|
||||
return kwargs["q"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer.gdn_prefill,
|
||||
"chunk_gated_delta_rule",
|
||||
fake_chunk_gated_delta_rule,
|
||||
)
|
||||
q = torch.zeros(1, 2, 1, 2)
|
||||
cu_seqlens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
|
||||
output, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=q,
|
||||
v=q,
|
||||
g=torch.zeros(1, 2, 1),
|
||||
beta=torch.zeros(1, 2, 1),
|
||||
initial_state=torch.zeros(1, 1, 2, 2),
|
||||
output_final_state=False,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
|
||||
assert captured_cu_seqlens is not None
|
||||
assert captured_cu_seqlens.dtype == torch.int64
|
||||
assert output.shape == q.shape
|
||||
assert final_state is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
),
|
||||
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
|
||||
)
|
||||
@pytest.mark.parametrize("use_cp", [True, False])
|
||||
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
|
||||
"""Each exported checkpoint is the state of the prefill stopped there.
|
||||
|
||||
The leading decode row is outside the prefill kernel. The two checkpointed
|
||||
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
|
||||
so the exporter must pick each state out of the shared emitted buffer;
|
||||
the third prefill takes no checkpoint. Both FlashInfer routes are covered
|
||||
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
|
||||
slots after the window, which the conv kernels leave alone.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
|
||||
num_spec = 2
|
||||
offsets = [0, 128, 0, 192]
|
||||
query_lens = [200, 50, 300]
|
||||
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
|
||||
num_blocks = 8
|
||||
|
||||
num_tokens = sum(query_lens)
|
||||
q = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
k = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
|
||||
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
initial_state = 0.1 * torch.randn(
|
||||
len(query_lens), num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
prefill_cu = [0]
|
||||
for query_len in query_lens:
|
||||
prefill_cu.append(prefill_cu[-1] + query_len)
|
||||
|
||||
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
|
||||
state_checkpoints = torch.empty(
|
||||
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor(prefill_cu, device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
|
||||
checkpoint_every_n_tokens=every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
)
|
||||
|
||||
# The exporter sees the decode row's token in front of the prefills.
|
||||
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
|
||||
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
|
||||
conv_state = torch.zeros(
|
||||
num_blocks,
|
||||
state_len + num_spec,
|
||||
conv_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).transpose(-1, -2)
|
||||
ssm_state = torch.zeros(
|
||||
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
|
||||
)
|
||||
MambaPrefillCheckpointExporter(state_len=state_len).export(
|
||||
MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
|
||||
state_indices=torch.tensor(state_indices, device=device),
|
||||
),
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
recurrent_rows=torch.tensor(rows, device=device),
|
||||
)
|
||||
|
||||
written = []
|
||||
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
|
||||
if not offset:
|
||||
continue
|
||||
written.append(block)
|
||||
seq = row - 1
|
||||
start = prefill_cu[seq]
|
||||
_, expected_state = fi_chunk_gated_delta_rule(
|
||||
q=q[:, start : start + offset],
|
||||
k=k[:, start : start + offset],
|
||||
v=v[:, start : start + offset],
|
||||
g=g[:, start : start + offset],
|
||||
beta=beta[:, start : start + offset],
|
||||
initial_state=initial_state[seq : seq + 1],
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor([0, offset], device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
|
||||
)
|
||||
end = cu_seqlens[row].item() + offset
|
||||
torch.testing.assert_close(
|
||||
conv_state[block, :, :state_len],
|
||||
raw_qkv[end - state_len : end].T,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
assert not conv_state[block, :, state_len:].any()
|
||||
|
||||
untouched = [i for i in range(num_blocks) if i not in written]
|
||||
assert not ssm_state[untouched].any()
|
||||
assert not conv_state[untouched].any()
|
||||
@@ -1,253 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layout dispatch tests for ``QwenGatedDeltaNetAttention._forward_core_rocm``.
|
||||
|
||||
The AITER fused reshape+conv kernel learned Qwen3.5's flat ``[q|k|v|z]``
|
||||
packing in https://github.com/ROCm/aiter/pull/3251, so flat-layout models take
|
||||
the decode fast path too instead of falling back to the generic path.
|
||||
|
||||
The dispatch tests run host-side with recording stubs. The prefill numerical
|
||||
tests exercise the production ChunkGatedDeltaRule wrapper on ROCm against a
|
||||
token-wise Torch reference, including ragged state and output-buffer layouts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
QwenGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
PREFIX = "model.layers.0.linear_attn"
|
||||
H = 2 # num key heads
|
||||
HV = 4 # num value heads
|
||||
K = 8 # head_k_dim
|
||||
V = 8 # head_v_dim
|
||||
|
||||
|
||||
def _make_metadata(
|
||||
*,
|
||||
num_prefills: int = 0,
|
||||
num_decodes: int = 2,
|
||||
spec_sequence_masks: torch.Tensor | None = None,
|
||||
):
|
||||
"""Only the fields the dispatch condition reads carry meaningful values."""
|
||||
return GDNAttentionMetadata(
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes,
|
||||
num_spec_decodes=0,
|
||||
num_spec_decode_tokens=0,
|
||||
num_actual_tokens=num_prefills + num_decodes,
|
||||
spec_sequence_masks=spec_sequence_masks,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(gqa_interleaved_layout: bool):
|
||||
"""Stub layer running the real ``_forward_core_rocm``, recording dispatch."""
|
||||
layer = types.SimpleNamespace()
|
||||
layer.prefix = PREFIX
|
||||
layer.gqa_interleaved_layout = gqa_interleaved_layout
|
||||
layer.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat"
|
||||
|
||||
layer.calls = []
|
||||
layer._forward_core_decode_aiter = lambda **kw: layer.calls.append("aiter")
|
||||
layer._forward_core = lambda **kw: layer.calls.append("generic")
|
||||
layer.prepare_gdn_attention_core_inputs = lambda qkvz, ba, n: (
|
||||
torch.zeros(n, H * K * 2 + HV * V),
|
||||
torch.zeros(n, HV, V),
|
||||
torch.zeros(n, HV),
|
||||
torch.zeros(n, HV),
|
||||
)
|
||||
layer._forward_core_rocm = types.MethodType(
|
||||
QwenGatedDeltaNetAttention._forward_core_rocm, layer
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def _run(layer, meta) -> str:
|
||||
num_tokens = max(meta.num_actual_tokens, 1)
|
||||
ctx = types.SimpleNamespace(attn_metadata={PREFIX: meta})
|
||||
with patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=ctx):
|
||||
layer._forward_core_rocm(
|
||||
qkvz=torch.zeros(num_tokens, 2 * H * K + 2 * HV * V),
|
||||
ba=torch.zeros(num_tokens, 2 * HV),
|
||||
z_out=torch.zeros(num_tokens, HV, V),
|
||||
core_attn_out=torch.zeros(num_tokens, HV, V),
|
||||
)
|
||||
assert len(layer.calls) == 1
|
||||
return layer.calls[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
|
||||
def test_pure_decode_takes_the_fast_path(gqa_interleaved_layout: bool) -> None:
|
||||
"""Both packings reach the fast path; flat used to be guarded out."""
|
||||
layer = _make_layer(gqa_interleaved_layout)
|
||||
assert _run(layer, _make_metadata()) == "aiter"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
|
||||
def test_layout_string_matches_the_packing(gqa_interleaved_layout: bool) -> None:
|
||||
"""The value handed to the kernel's ``qkvz_layout`` parameter."""
|
||||
layer = _make_layer(gqa_interleaved_layout)
|
||||
expected = "interleaved" if gqa_interleaved_layout else "flat"
|
||||
assert layer.qkvz_layout == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gqa_interleaved_layout", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"meta_kwargs",
|
||||
[
|
||||
{"num_prefills": 1},
|
||||
{"num_decodes": 0},
|
||||
{"spec_sequence_masks": torch.zeros(1, dtype=torch.bool)},
|
||||
],
|
||||
ids=["has_prefill", "no_decode", "spec_decode"],
|
||||
)
|
||||
def test_non_pure_decode_batches_use_the_generic_path(
|
||||
gqa_interleaved_layout: bool, meta_kwargs: dict
|
||||
) -> None:
|
||||
layer = _make_layer(gqa_interleaved_layout)
|
||||
assert _run(layer, _make_metadata(**meta_kwargs)) == "generic"
|
||||
|
||||
|
||||
def _prefill_reference(q, k, v, g, beta, initial_state, lengths):
|
||||
"""Independent token-wise recurrence in the layer's [value, key] layout."""
|
||||
q, k, v, g, beta, initial_state = (
|
||||
x.cpu().float() for x in (q, k, v, g, beta, initial_state)
|
||||
)
|
||||
q = q.repeat_interleave(v.shape[2] // q.shape[2], dim=2)
|
||||
k = k.repeat_interleave(v.shape[2] // k.shape[2], dim=2)
|
||||
q = q * q.shape[-1] ** -0.5
|
||||
output = torch.empty_like(v)
|
||||
final_state = torch.empty_like(initial_state)
|
||||
previous_threads = torch.get_num_threads()
|
||||
torch.set_num_threads(1)
|
||||
try:
|
||||
start = 0
|
||||
for sequence, length in enumerate(lengths):
|
||||
state = initial_state[sequence].clone()
|
||||
for token in range(start, start + length):
|
||||
state = state * g[0, token].exp()[:, None, None]
|
||||
key = k[0, token]
|
||||
prediction = (state * key[:, None, :]).sum(dim=-1)
|
||||
delta = (v[0, token] - prediction) * beta[0, token, :, None]
|
||||
state = state + delta[:, :, None] * key[:, None, :]
|
||||
output[0, token] = (state * q[0, token, :, None, :]).sum(dim=-1)
|
||||
final_state[sequence] = state
|
||||
start += length
|
||||
finally:
|
||||
torch.set_num_threads(previous_threads)
|
||||
return output, final_state
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state_dtype,correlated",
|
||||
[(torch.bfloat16, False), (torch.float32, False), (torch.float32, True)],
|
||||
ids=["ragged-bf16-state", "ragged-fp32-state", "correlated-keys"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
ChunkGatedDeltaRule,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.index import (
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("ROCm prefill numerics require a GPU")
|
||||
# Only the config is needed for production backend selection; no weights.
|
||||
config = VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
revision="2fc06364715b967f1860aea9cf38778875588b17",
|
||||
max_model_len=1024,
|
||||
),
|
||||
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
|
||||
additional_config={"gdn_prefill_backend": "triton"},
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
prefill = ChunkGatedDeltaRule()
|
||||
assert prefill.gdn_prefill_backend == "triton"
|
||||
assert prefill._forward_method == prefill.forward_native
|
||||
|
||||
torch.manual_seed(1234)
|
||||
lengths = [64] if correlated else [1, 63, 64, 65, 129]
|
||||
tokens, key_heads, value_heads, dim = sum(lengths), 4, 8, 128
|
||||
q = torch.randn(1, tokens, key_heads, dim, device="cuda")
|
||||
q = torch.nn.functional.normalize(q, dim=-1).to(torch.bfloat16)
|
||||
k = torch.randn_like(q)
|
||||
k = torch.nn.functional.normalize(k.float(), dim=-1).to(q.dtype)
|
||||
v = torch.randn(1, tokens, value_heads, dim, device="cuda", dtype=q.dtype)
|
||||
g = -torch.rand(1, tokens, value_heads, device="cuda") * 0.1
|
||||
beta = torch.rand_like(g)
|
||||
state = (torch.randn(len(lengths), value_heads, dim, dim, device="cuda") * 0.05).to(
|
||||
state_dtype
|
||||
)
|
||||
if correlated:
|
||||
q.zero_()
|
||||
k.zero_()
|
||||
q[..., 0] = 1
|
||||
k[..., 0] = 1
|
||||
g.zero_()
|
||||
beta.fill_(0.9)
|
||||
state.zero_()
|
||||
expected, expected_state = _prefill_reference(q, k, v, g, beta, state, lengths)
|
||||
initial_state_copy = state.clone()
|
||||
cu_seqlens = torch.tensor(
|
||||
[0, *torch.tensor(lengths).cumsum(0).tolist()],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
kwargs = dict(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=prepare_chunk_indices(cu_seqlens, 64),
|
||||
chunk_offsets=prepare_chunk_offsets(cu_seqlens, 64),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
storage = torch.full(
|
||||
(tokens + 3, value_heads, dim), float("nan"), device="cuda", dtype=q.dtype
|
||||
)
|
||||
output, final_state = prefill(**kwargs, core_attn_out=storage[:tokens])
|
||||
assert output.data_ptr() == storage.data_ptr()
|
||||
assert output.shape == v.shape
|
||||
assert final_state.shape == state.shape
|
||||
assert torch.isnan(storage[tokens:]).all()
|
||||
torch.testing.assert_close(state, initial_state_copy, atol=0, rtol=0)
|
||||
|
||||
# Preserve the upstream prefill suite's numerical budgets. The reference
|
||||
# here is a token-wise recurrence, independent of the chunk implementation.
|
||||
output_error = (output.cpu().float() - expected).abs()
|
||||
state_error = (final_state.cpu().float() - expected_state).abs()
|
||||
assert torch.isfinite(output).all() and torch.isfinite(final_state).all()
|
||||
assert output_error.max() < (1e-1 if correlated else 2e-3)
|
||||
assert output_error.mean() < (2e-3 if correlated else 6e-5)
|
||||
assert state_error.max() < (2e-1 if correlated else 2e-2)
|
||||
assert state_error.mean() < 6e-4
|
||||
|
||||
no_buffer_output, no_buffer_state = prefill(**kwargs)
|
||||
torch.testing.assert_close(no_buffer_output, output, atol=0, rtol=0)
|
||||
torch.testing.assert_close(no_buffer_state, final_state, atol=0, rtol=0)
|
||||
@@ -1,169 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""An SSD intermediate state equals an independent run over the same prefix.
|
||||
|
||||
This is the central claim of internal prefill checkpoints: the state the SSD
|
||||
scan already materializes at the checkpoint's chunk boundary is exactly the
|
||||
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
|
||||
cached and later resumed from. The last test covers the other half: moving
|
||||
that state, and the conv window, into the paged cache.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
|
||||
store_prefill_checkpoint,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
|
||||
|
||||
|
||||
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
|
||||
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
|
||||
|
||||
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
|
||||
state when a checkpoint was requested, else the per-sequence final state.
|
||||
"""
|
||||
A, dt, X, B, C = inputs
|
||||
offsets = [checkpoint_offset] if checkpoint_offset else None
|
||||
cu, seq_idx, last, ckpt = compute_chunk_metadata(
|
||||
chunk_size,
|
||||
1,
|
||||
torch.tensor([0]),
|
||||
torch.tensor([0, seqlen]),
|
||||
checkpoint_offsets_p=offsets,
|
||||
)
|
||||
|
||||
x = X[:seqlen]
|
||||
out = torch.empty_like(x)
|
||||
states = mamba_chunk_scan_combined_varlen(
|
||||
x,
|
||||
dt[:seqlen],
|
||||
A,
|
||||
B[:seqlen],
|
||||
C[:seqlen],
|
||||
chunk_size,
|
||||
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
|
||||
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
|
||||
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
|
||||
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
|
||||
out=out,
|
||||
D=None,
|
||||
return_intermediate_states=checkpoint_offset > 0,
|
||||
state_dtype=torch.float32,
|
||||
)
|
||||
return states, ckpt[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
|
||||
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
|
||||
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
|
||||
|
||||
48 is deliberately not a multiple of chunk_size: align-mode block sizes
|
||||
carry no chunk_size factor, so off-grid checkpoints are the common case
|
||||
and are what the chunk split exists to support.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
|
||||
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
|
||||
checkpoint_state = states[ckpt_idx]
|
||||
|
||||
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
checkpoint_state.to(torch.float32),
|
||||
prefix_final[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_does_not_perturb_the_final_state():
|
||||
"""Exporting a checkpoint must not change what the prefill produces.
|
||||
|
||||
The chunk split is an extra boundary, not a different computation, so the
|
||||
sequence's final state has to match a run that took no checkpoint.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
|
||||
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
with_ckpt[-1].to(torch.float32),
|
||||
without_ckpt[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec_slots", [0, 3])
|
||||
@pytest.mark.parametrize("dim_first", [True, False])
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
|
||||
"""The store reads the checkpoint row out of every chunk state.
|
||||
|
||||
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
|
||||
program's own row, so the store only passes if ``chunk_idx`` is used.
|
||||
Row 1 declines and must write nothing. The conv input is a column slice of
|
||||
a wider projection, the SD conv layout is a transposed view, and SSM rows
|
||||
are padded pages, matching the strides the mixer passes. Spec decoding
|
||||
widens the conv state, and the store must still fill only the first
|
||||
``state_len`` slots with the newest inputs.
|
||||
"""
|
||||
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
|
||||
projection = torch.randn(64, dim + 8, device=DEVICE)
|
||||
conv_input = projection[:, :dim]
|
||||
num_slots = state_len + spec_slots
|
||||
if dim_first:
|
||||
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
|
||||
else:
|
||||
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
|
||||
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
|
||||
# Blocks are pages holding more than the SSM state, so rows are padded.
|
||||
row = heads * head_dim * d_state
|
||||
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
|
||||
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
|
||||
state_indices=torch.tensor(
|
||||
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
offsets=[32, 0],
|
||||
)
|
||||
|
||||
store_prefill_checkpoint(
|
||||
checkpoint,
|
||||
torch.tensor([3, 0], device=DEVICE),
|
||||
conv_input.T,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
|
||||
state_len=state_len,
|
||||
)
|
||||
|
||||
exact = dict(rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
|
||||
)
|
||||
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
|
||||
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
|
||||
assert not conv_state[:2].any() and not ssm_storage[:2].any()
|
||||
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,627 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange, repeat
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
|
||||
_chunk_scan_fwd,
|
||||
_chunk_scan_fwd_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
|
||||
|
||||
# All kernels exercised here are pure Triton, so they run on any backend
|
||||
# that the vLLM platform layer treats as a CUDA-alike device or as XPU.
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Mamba2 SSD Triton kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
|
||||
# Added by the IBM Team, 2024
|
||||
|
||||
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py
|
||||
|
||||
|
||||
# this is the segsum implementation taken from above
|
||||
def segsum(x):
|
||||
"""Calculates segment sum."""
|
||||
T = x.size(-1)
|
||||
x = repeat(x, "... d -> ... d e", e=T)
|
||||
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=-1)
|
||||
x = x.masked_fill(~mask, 0)
|
||||
x_segsum = torch.cumsum(x, dim=-2)
|
||||
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=0)
|
||||
x_segsum = x_segsum.masked_fill(~mask, -torch.inf)
|
||||
return x_segsum
|
||||
|
||||
|
||||
def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
|
||||
"""Arguments:
|
||||
X: (batch, length, n_heads, d_head)
|
||||
A: (batch, length, n_heads)
|
||||
B: (batch, length, n_heads, d_state)
|
||||
C: (batch, length, n_heads, d_state)
|
||||
|
||||
Return:
|
||||
Y: (batch, length, n_heads, d_head)
|
||||
|
||||
"""
|
||||
assert X.dtype == A.dtype == B.dtype == C.dtype
|
||||
assert X.shape[1] % block_len == 0
|
||||
|
||||
# Rearrange into blocks/chunks
|
||||
X, A, B, C = (
|
||||
rearrange(x, "b (c l) ... -> b c l ...", l=block_len) for x in (X, A, B, C)
|
||||
)
|
||||
|
||||
A = rearrange(A, "b c l h -> b h c l")
|
||||
A_cumsum = torch.cumsum(A, dim=-1)
|
||||
|
||||
# 1. Compute the output for each intra-chunk (diagonal blocks)
|
||||
L = torch.exp(segsum(A))
|
||||
Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, X)
|
||||
|
||||
# 2. Compute the state for each intra-chunk
|
||||
# (right term of low-rank factorization of off-diagonal blocks; B terms)
|
||||
decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum)
|
||||
states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, X)
|
||||
|
||||
# 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at
|
||||
# chunk boundaries
|
||||
# (middle term of factorization of off-diag blocks; A terms)
|
||||
if initial_states is None:
|
||||
initial_states = torch.zeros_like(states[:, :1])
|
||||
states = torch.cat([initial_states, states], dim=1)
|
||||
decay_chunk = torch.exp(segsum(F.pad(A_cumsum[:, :, :, -1], (1, 0))))
|
||||
new_states = torch.einsum("bhzc,bchpn->bzhpn", decay_chunk, states)
|
||||
states, final_state = new_states[:, :-1], new_states[:, -1]
|
||||
|
||||
# 4. Compute state -> output conversion per chunk
|
||||
# (left term of low-rank factorization of off-diagonal blocks; C terms)
|
||||
state_decay_out = torch.exp(A_cumsum)
|
||||
Y_off = torch.einsum("bclhn,bchpn,bhcl->bclhp", C, states, state_decay_out)
|
||||
|
||||
# Add output of intra-chunk and inter-chunk terms
|
||||
# (diagonal and off-diagonal blocks)
|
||||
Y = rearrange(Y_diag + Y_off, "b c l h p -> b (c l) h p")
|
||||
return Y, final_state
|
||||
|
||||
|
||||
def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device=DEVICE):
|
||||
set_random_seed(0)
|
||||
A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device))
|
||||
dt = F.softplus(
|
||||
torch.randn(batch_size, seqlen, n_heads, dtype=itype, device=device) - 4
|
||||
)
|
||||
X = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
|
||||
B = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
|
||||
C = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device)
|
||||
|
||||
return A, dt, X, B, C
|
||||
|
||||
|
||||
def generate_continuous_batched_examples(
|
||||
example_lens_by_batch,
|
||||
num_examples,
|
||||
full_length,
|
||||
last_taken,
|
||||
exhausted,
|
||||
n_heads,
|
||||
d_head,
|
||||
itype,
|
||||
device=DEVICE,
|
||||
return_naive_ref=True,
|
||||
):
|
||||
# this function generates a random examples of certain length
|
||||
# and then cut according to "example_lens_by_batch" and feed
|
||||
# them in continuous batches to the kernels.
|
||||
# If if return_naive_ref=True, the naive torch implementation
|
||||
# ssd_minimal_discrete will be used to compute and return
|
||||
# reference output.
|
||||
|
||||
# generate the full-length example
|
||||
A, dt, X, B, C = generate_random_inputs(
|
||||
num_examples, full_length, n_heads, d_head, itype
|
||||
)
|
||||
|
||||
if return_naive_ref:
|
||||
Y_min, final_state_min = ssd_minimal_discrete(
|
||||
X * dt.unsqueeze(-1), A * dt, B, C, block_len=full_length // 4
|
||||
)
|
||||
|
||||
# internal function that outputs a cont batch of examples
|
||||
# given a tuple of lengths for each example in the batch
|
||||
# e.g., example_lens=(8, 4) means take 8 samples from first eg,
|
||||
# 4 examples from second eg, etc
|
||||
def get_continuous_batch(example_lens: tuple[int, ...]):
|
||||
indices = []
|
||||
for i, x in enumerate(example_lens):
|
||||
c = last_taken.get(i, 0)
|
||||
indices.append((c, c + x))
|
||||
last_taken[i] = (c + x) % full_length
|
||||
exhausted[i] = last_taken[i] == 0
|
||||
|
||||
return (
|
||||
torch.concat([x[i, s:e] for i, (s, e) in enumerate(indices)]).unsqueeze(0)
|
||||
for x in (dt, X, B, C)
|
||||
)
|
||||
|
||||
# internal function that maps "n" to the appropriate right boundary
|
||||
# value when forming continuous batches from examples of length given
|
||||
# by "full_length".
|
||||
# - e.g., when n > full_length, returns n % full_length
|
||||
# when n == full_length, returns full_length
|
||||
def end_boundary(n: int):
|
||||
return n - ((n - 1) // full_length) * full_length
|
||||
|
||||
IND_E = None
|
||||
for spec in example_lens_by_batch:
|
||||
# get the (maybe partial) example seen in this cont batch
|
||||
dt2, X2, B2, C2 = get_continuous_batch(spec)
|
||||
|
||||
# get the metadata
|
||||
cu_seqlens = torch.tensor((0,) + spec, device=device).cumsum(dim=0)
|
||||
seq_idx = torch.zeros(
|
||||
cu_seqlens[-1], dtype=torch.int32, device=cu_seqlens.device
|
||||
)
|
||||
for i, (srt, end) in enumerate(
|
||||
zip(
|
||||
cu_seqlens,
|
||||
cu_seqlens[1:],
|
||||
)
|
||||
):
|
||||
seq_idx[srt:end] = i
|
||||
|
||||
# for cont batch
|
||||
if IND_E is None:
|
||||
IND_S = [0 for _ in range(len(spec))]
|
||||
else:
|
||||
IND_S = [x % full_length for x in IND_E]
|
||||
IND_E = [end_boundary(x + y) for x, y in zip(IND_S, spec)]
|
||||
|
||||
# varlen has implicit batch=1
|
||||
dt2 = dt2.squeeze(0)
|
||||
X2 = X2.squeeze(0)
|
||||
B2 = B2.squeeze(0)
|
||||
C2 = C2.squeeze(0)
|
||||
yield (
|
||||
[Y_min[s, IND_S[s] : IND_E[s]] for s in range(num_examples)]
|
||||
if return_naive_ref
|
||||
else None,
|
||||
cu_seqlens,
|
||||
seq_idx,
|
||||
(A, dt2, X2, B2, C2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n_heads", [4, 16, 32])
|
||||
@pytest.mark.parametrize("d_head", [5, 8, 32, 128])
|
||||
@pytest.mark.parametrize("seq_len_chunk_size", [(112, 16), (128, 32)])
|
||||
def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, itype):
|
||||
# this tests the kernels on a single example (bs=1)
|
||||
|
||||
# TODO: the bfloat16 case requires higher thresholds. To be investigated
|
||||
|
||||
if itype == torch.bfloat16:
|
||||
atol, rtol = 5e-2, 5e-2
|
||||
else:
|
||||
atol, rtol = 8e-3, 5e-3
|
||||
|
||||
# set seed
|
||||
batch_size = 1 # batch_size
|
||||
# ssd_minimal_discrete requires chunk_size divide seqlen
|
||||
# - this is only required for generating the reference seqs,
|
||||
# it is not an operational limitation.
|
||||
seqlen, chunk_size = seq_len_chunk_size
|
||||
|
||||
A, dt, X, B, C = generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype)
|
||||
|
||||
Y_min, final_state_min = ssd_minimal_discrete(
|
||||
X * dt.unsqueeze(-1), A * dt, B, C, chunk_size
|
||||
)
|
||||
|
||||
cu_seqlens = torch.tensor((0, seqlen), device=DEVICE).cumsum(dim=0)
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
|
||||
)
|
||||
# varlen has implicit batch=1
|
||||
X = X.squeeze(0)
|
||||
dt = dt.squeeze(0)
|
||||
A = A.squeeze(0)
|
||||
B = B.squeeze(0)
|
||||
C = C.squeeze(0)
|
||||
Y = torch.empty_like(X)
|
||||
final_state = mamba_chunk_scan_combined_varlen(
|
||||
X,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
cu_seqlens=cu_seqlens.to(torch.int32),
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx_chunks,
|
||||
out=Y,
|
||||
D=None,
|
||||
)
|
||||
|
||||
# just test the last in sequence
|
||||
torch.testing.assert_close(Y[-1], Y_min[0, -1], atol=atol, rtol=rtol)
|
||||
|
||||
# just test the last head
|
||||
# NOTE, in the kernel we always cast states to fp32
|
||||
torch.testing.assert_close(
|
||||
final_state[:, -1].to(torch.float32),
|
||||
final_state_min[:, -1].to(torch.float32),
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.float32])
|
||||
@pytest.mark.parametrize("n_heads", [4, 8])
|
||||
@pytest.mark.parametrize("d_head", [5, 16, 32])
|
||||
@pytest.mark.parametrize(
|
||||
"seq_len_chunk_size_cases",
|
||||
[
|
||||
# small-ish chunk_size (8)
|
||||
(64, 8, 2, [(64, 32), (64, 32)]),
|
||||
(64, 8, 2, [(8, 8), (8, 8), (8, 8)]), # chunk size boundary
|
||||
(
|
||||
64,
|
||||
8,
|
||||
2,
|
||||
[(4, 4), (4, 4), (4, 4), (4, 4)],
|
||||
), # chunk_size larger than cont batches
|
||||
(64, 8, 5, [(64, 32, 16, 8, 8)]),
|
||||
# large-ish chunk_size (256)
|
||||
(64, 256, 1, [(5,), (1,), (1,), (1,)]), # irregular sizes with small sequences
|
||||
(
|
||||
64,
|
||||
256,
|
||||
2,
|
||||
[(5, 30), (1, 2), (1, 2), (1, 2)],
|
||||
), # irregular sizes with small sequences
|
||||
# we also need to test some large seqlen
|
||||
# to catch errors with init states decay
|
||||
(768, 128, 2, [(138, 225), (138, 225)]),
|
||||
],
|
||||
)
|
||||
def test_mamba_chunk_scan_cont_batch(d_head, n_heads, seq_len_chunk_size_cases, itype):
|
||||
# this test with multiple examples in a continuous batch
|
||||
# (i.e. chunked prefill)
|
||||
|
||||
seqlen, chunk_size, num_examples, cases = seq_len_chunk_size_cases
|
||||
|
||||
# This test can have larger error for longer sequences
|
||||
if seqlen > 256:
|
||||
atol, rtol = 1e-2, 5e-3
|
||||
else:
|
||||
atol, rtol = 5e-3, 5e-3
|
||||
|
||||
# hold state during the cutting process so we know if an
|
||||
# example has been exhausted and needs to cycle
|
||||
last_taken: dict = {} # map: eg -> pointer to last taken sample
|
||||
exhausted: dict = {} # map: eg -> boolean indicating example is exhausted
|
||||
|
||||
states = None
|
||||
for Y_min, cu_seqlens, _token_seq_idx, (
|
||||
A,
|
||||
dt,
|
||||
X,
|
||||
B,
|
||||
C,
|
||||
) in generate_continuous_batched_examples(
|
||||
cases, num_examples, seqlen, last_taken, exhausted, n_heads, d_head, itype
|
||||
):
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
|
||||
)
|
||||
|
||||
Y = torch.empty_like(X)
|
||||
new_states = mamba_chunk_scan_combined_varlen(
|
||||
X,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
cu_seqlens=cu_seqlens.to(torch.int32),
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx_chunks,
|
||||
out=Y,
|
||||
D=None,
|
||||
initial_states=states,
|
||||
)
|
||||
|
||||
# just test the last in sequence
|
||||
for i in range(num_examples):
|
||||
# just test one dim and dstate
|
||||
Y_eg = Y[cu_seqlens[i] : cu_seqlens[i + 1], 0, 0]
|
||||
Y_min_eg = Y_min[i][:, 0, 0]
|
||||
torch.testing.assert_close(Y_eg, Y_min_eg, atol=atol, rtol=rtol)
|
||||
|
||||
# update states
|
||||
states = new_states
|
||||
for i, clear in exhausted.items():
|
||||
if clear:
|
||||
states[i].fill_(0.0)
|
||||
exhausted[i] = False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [8, 256])
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[(16, 20), (270, 88, 212, 203)],
|
||||
)
|
||||
def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
|
||||
# This test verifies the correctness of the chunked prefill implementation
|
||||
# in the mamba2 ssd kernels, by comparing concatenation (in the sequence
|
||||
# dimension) of chunked results with the full sequence result.
|
||||
# It is different from test_mamba_chunk_scan_cont_batch by:
|
||||
# 1. Not using the naive torch implementation (ssd_minimal_discrete) to get
|
||||
# reference outputs. Instead, it compares chunked kernel outputs to full
|
||||
# sequence kernel outputs. This is the most straightforward way to
|
||||
# assert chunked prefill correctness.
|
||||
# 2. It focuses on cases where sequences change in the middle of mamba
|
||||
# chunks, and not necessarily on chunk boundaries.
|
||||
|
||||
max_seqlen = max(seqlens)
|
||||
# This test can have larger error for longer sequences
|
||||
if max_seqlen > 256:
|
||||
atol, rtol = 1e-2, 5e-3
|
||||
else:
|
||||
atol, rtol = 5e-3, 5e-3
|
||||
|
||||
num_sequences = len(seqlens)
|
||||
n_heads = 16
|
||||
d_head = 64
|
||||
itype = torch.float32
|
||||
|
||||
# hold state during the cutting process so we know if an
|
||||
# example has been exhausted and needs to cycle
|
||||
last_taken: dict = {} # map: eg -> pointer to last taken sample
|
||||
exhausted: dict = {} # map: eg -> boolean indicating example is exhausted
|
||||
_, cu_seqlens, seq_idx, (A, dt, X, B, C) = next(
|
||||
generate_continuous_batched_examples(
|
||||
[seqlens],
|
||||
num_sequences,
|
||||
max_seqlen,
|
||||
last_taken,
|
||||
exhausted,
|
||||
n_heads,
|
||||
d_head,
|
||||
itype,
|
||||
return_naive_ref=False,
|
||||
)
|
||||
)
|
||||
seqlens = torch.tensor(seqlens, dtype=torch.int32, device=X.device)
|
||||
device = X.device
|
||||
|
||||
## full seqlen computation
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(cu_seqlens, chunk_size)
|
||||
)
|
||||
Y_ref = torch.empty_like(X)
|
||||
state_ref = mamba_chunk_scan_combined_varlen(
|
||||
X,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
cu_seqlens=cu_seqlens.to(torch.int32),
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx_chunks,
|
||||
out=Y_ref,
|
||||
D=None,
|
||||
initial_states=None,
|
||||
)
|
||||
|
||||
## chunked seqlen computation
|
||||
# first chunk
|
||||
chunked_seqlens = seqlens // 2
|
||||
chunked_cu_seqlens = torch.cat(
|
||||
[torch.tensor([0], device=device), torch.cumsum(chunked_seqlens, dim=0)], dim=0
|
||||
)
|
||||
chunked_input_seq_len = chunked_cu_seqlens[-1]
|
||||
X_chunked = torch.zeros_like(X)[:chunked_input_seq_len, ...]
|
||||
dt_chunked = torch.zeros_like(dt)[:chunked_input_seq_len, ...]
|
||||
B_chunked = torch.zeros_like(B)[:chunked_input_seq_len, ...]
|
||||
C_chunked = torch.zeros_like(C)[:chunked_input_seq_len, ...]
|
||||
for i in range(num_sequences):
|
||||
chunk_f = lambda x, i: x[
|
||||
cu_seqlens[i] : cu_seqlens[i] + chunked_seqlens[i], ...
|
||||
]
|
||||
|
||||
X_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
|
||||
X, i
|
||||
)
|
||||
dt_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
|
||||
dt, i
|
||||
)
|
||||
B_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
|
||||
B, i
|
||||
)
|
||||
C_chunked[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...] = chunk_f(
|
||||
C, i
|
||||
)
|
||||
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(chunked_cu_seqlens, chunk_size)
|
||||
)
|
||||
Y_partial = torch.empty_like(X_chunked)
|
||||
partial_state = mamba_chunk_scan_combined_varlen(
|
||||
X_chunked,
|
||||
dt_chunked,
|
||||
A,
|
||||
B_chunked,
|
||||
C_chunked,
|
||||
chunk_size,
|
||||
cu_seqlens=chunked_cu_seqlens.to(torch.int32),
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx_chunks,
|
||||
out=Y_partial,
|
||||
D=None,
|
||||
initial_states=None,
|
||||
)
|
||||
|
||||
# remaining chunk
|
||||
remaining_chunked_seqlens = seqlens - chunked_seqlens
|
||||
remaining_chunked_cu_seqlens = torch.cat(
|
||||
[
|
||||
torch.tensor([0], device=device),
|
||||
torch.cumsum(remaining_chunked_seqlens, dim=0),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
remaining_chunked_input_seq_len = remaining_chunked_cu_seqlens[-1]
|
||||
remaining_X_chunked = torch.zeros_like(X)[:remaining_chunked_input_seq_len, ...]
|
||||
remaining_dt_chunked = torch.zeros_like(dt)[:remaining_chunked_input_seq_len, ...]
|
||||
remaining_B_chunked = torch.zeros_like(B)[:remaining_chunked_input_seq_len, ...]
|
||||
remaining_C_chunked = torch.zeros_like(C)[:remaining_chunked_input_seq_len, ...]
|
||||
for i in range(num_sequences):
|
||||
remaining_chunk_f = lambda x, i: x[
|
||||
cu_seqlens[i] + chunked_seqlens[i] : cu_seqlens[i + 1], ...
|
||||
]
|
||||
|
||||
remaining_X_chunked[
|
||||
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
|
||||
] = remaining_chunk_f(X, i)
|
||||
remaining_dt_chunked[
|
||||
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
|
||||
] = remaining_chunk_f(dt, i)
|
||||
remaining_B_chunked[
|
||||
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
|
||||
] = remaining_chunk_f(B, i)
|
||||
remaining_C_chunked[
|
||||
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1], ...
|
||||
] = remaining_chunk_f(C, i)
|
||||
|
||||
# assert input chunking is correct
|
||||
concat_chunk_f = lambda pt1, pt2, i: torch.cat(
|
||||
[
|
||||
pt1[chunked_cu_seqlens[i] : chunked_cu_seqlens[i + 1], ...],
|
||||
pt2[
|
||||
remaining_chunked_cu_seqlens[i] : remaining_chunked_cu_seqlens[i + 1],
|
||||
...,
|
||||
],
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
concat_batch_f = lambda pt1, pt2: torch.cat(
|
||||
[concat_chunk_f(pt1, pt2, i) for i in range(num_sequences)], dim=0
|
||||
)
|
||||
|
||||
assert concat_batch_f(X_chunked, remaining_X_chunked).equal(X)
|
||||
assert concat_batch_f(dt_chunked, remaining_dt_chunked).equal(dt)
|
||||
assert concat_batch_f(B_chunked, remaining_B_chunked).equal(B)
|
||||
assert concat_batch_f(C_chunked, remaining_C_chunked).equal(C)
|
||||
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx_chunks = (
|
||||
compute_varlen_chunk_metadata(remaining_chunked_cu_seqlens, chunk_size)
|
||||
)
|
||||
|
||||
Y_chunked = torch.empty_like(remaining_X_chunked)
|
||||
state_chunked = mamba_chunk_scan_combined_varlen(
|
||||
remaining_X_chunked,
|
||||
remaining_dt_chunked,
|
||||
A,
|
||||
remaining_B_chunked,
|
||||
remaining_C_chunked,
|
||||
chunk_size,
|
||||
cu_seqlens=remaining_chunked_cu_seqlens.to(torch.int32),
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx_chunks,
|
||||
out=Y_chunked,
|
||||
D=None,
|
||||
initial_states=partial_state,
|
||||
)
|
||||
Y = concat_batch_f(Y_partial, Y_chunked)
|
||||
|
||||
# kernel chunked is same as kernel overall
|
||||
for i in range(num_sequences):
|
||||
Y_seq = Y[cu_seqlens[i] : cu_seqlens[i + 1], ...]
|
||||
Y_ref_seq = Y_ref[cu_seqlens[i] : cu_seqlens[i + 1], ...]
|
||||
torch.testing.assert_close(
|
||||
Y_seq[: chunked_seqlens[i], ...],
|
||||
Y_ref_seq[: chunked_seqlens[i], ...],
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} output part1 " + x,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
Y_seq[chunked_seqlens[i] :, ...],
|
||||
Y_ref_seq[chunked_seqlens[i] :, ...],
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} output part2 " + x,
|
||||
)
|
||||
|
||||
state_seq = state_chunked[i]
|
||||
state_seq_ref = state_ref[i]
|
||||
torch.testing.assert_close(
|
||||
state_seq,
|
||||
state_seq_ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} state " + x,
|
||||
)
|
||||
|
||||
|
||||
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
|
||||
"""The autotuner picks a config by speed only, so every config in the list
|
||||
must give the same finite output. On sm90, configs that pipeline a wgmma
|
||||
with `cb` in registers wrote NaN into whole rows (#61603)."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
set_random_seed(0)
|
||||
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
|
||||
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
|
||||
seqlen, nchunks = 181, 2
|
||||
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
|
||||
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
|
||||
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
|
||||
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
|
||||
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
|
||||
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
|
||||
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
|
||||
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
|
||||
|
||||
ref = None
|
||||
for config in list(_chunk_scan_fwd_kernel.configs):
|
||||
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
|
||||
for _ in range(10):
|
||||
out = torch.empty_like(x)
|
||||
try:
|
||||
_chunk_scan_fwd(
|
||||
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
|
||||
)
|
||||
except OutOfResources:
|
||||
# The autotuner skips configs that do not fit this device.
|
||||
break
|
||||
assert torch.isfinite(out).all(), f"non-finite output with {config}"
|
||||
if ref is None:
|
||||
ref = out
|
||||
torch.testing.assert_close(
|
||||
out,
|
||||
ref,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
msg=lambda m, config=config: f"{config}: {m}",
|
||||
)
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for ``_memcpy_u64_tiled``.
|
||||
|
||||
Guards the head/body/tail alignment handling and the u64 body's tile
|
||||
partitioning used by ``postprocess_mamba_fused_kernel`` and
|
||||
``precopy_mamba_align_fused_kernel``. The device function must be
|
||||
byte-identical to a ``dst[dst_off:dst_off+copy_size] = src[src_off:src_off+
|
||||
copy_size]`` slice copy for every combination of:
|
||||
|
||||
* ``copy_size``: the degenerate 0-byte case, sub-8B (head-only), 8B-aligned
|
||||
bodies, and a multi-tile body that spans several ``COPY_BLOCK_SIZE``
|
||||
iterations.
|
||||
* ``(src_off, dst_off)`` pairs: aligned, shared sub-8B misalignment (fast
|
||||
path exercising head/tail masking), and mismatched src/dst alignment
|
||||
(byte-fallback path). The kernel branches on ``(src ^ dst) & 7``, so a
|
||||
full 5x5 product would re-exercise the same two codepaths.
|
||||
* ``NUM_TILES``: 1 (SD conv callsite, single-CTA memcpy) and
|
||||
``_TEMPORAL_TILES`` (temporal callsite, u64 range partitioned across
|
||||
CTAs).
|
||||
* ``COPY_BLOCK_SIZE``: the production value 1024 (single-tile at these
|
||||
sizes) and a small value 8 so the existing ``copy_size`` cases actually
|
||||
cross tile boundaries under ``NUM_TILES=_TEMPORAL_TILES``. The
|
||||
partitioning math is COPY_BLOCK_SIZE-agnostic, so testing at a small
|
||||
value covers boundary arithmetic for all values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, _memcpy_u64_tiled
|
||||
|
||||
try:
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="_memcpy_u64_tiled needs CUDA/Triton",
|
||||
)
|
||||
_parametrize = pytest.mark.parametrize
|
||||
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
|
||||
pytest = None
|
||||
|
||||
def _parametrize(_name, _values):
|
||||
def _deco(fn):
|
||||
return fn
|
||||
|
||||
return _deco
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _memcpy_wrapper_kernel(
|
||||
src_ptr,
|
||||
dst_ptr,
|
||||
copy_size,
|
||||
src_off,
|
||||
dst_off,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
):
|
||||
"""Thin harness: one CTA per tile, exercising the device function."""
|
||||
tile_idx = tl.program_id(0)
|
||||
src_addr = src_ptr.to(tl.int64) + src_off
|
||||
dst_addr = dst_ptr.to(tl.int64) + dst_off
|
||||
_memcpy_u64_tiled(
|
||||
src_addr,
|
||||
dst_addr,
|
||||
copy_size,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE=COPY_BLOCK_SIZE,
|
||||
NUM_TILES=NUM_TILES,
|
||||
)
|
||||
|
||||
|
||||
# Copy sizes: 0/1/7 (all-head), 8 (all-body when dst-aligned), 15 (body+tail,
|
||||
# tail-mask path), 16 (small tiled body), 1024 and 4 KiB (spans all 16 tiles
|
||||
# at COPY_BLOCK_SIZE=8, with 4 KiB adding multi-iteration-per-tile coverage;
|
||||
# both single-tile at COPY_BLOCK_SIZE=1024).
|
||||
_COPY_SIZES = [0, 1, 7, 8, 15, 16, 1024, 4 * 1024]
|
||||
# (src_off, dst_off) pairs. Torch tensors are 256B-aligned at data_ptr, so
|
||||
# slicing by these bytes yields a controlled sub-8B alignment. The kernel
|
||||
# branches on ``(src ^ dst) & 7``, so we cover both sides plus the aligned
|
||||
# baseline; the full 5x5 product added no coverage.
|
||||
_ALIGN_PAIRS = [
|
||||
(0, 0), # fully aligned; head/tail masked out
|
||||
(3, 3), # shared misalignment; fast path with head_bytes=5
|
||||
(7, 7), # shared, extreme; head_bytes=1 with non-empty body
|
||||
(1, 3), # mismatched; byte load/store fallback
|
||||
(3, 1), # mismatched, opposite direction
|
||||
]
|
||||
_MAX_ALIGN_OFF = max(o for pair in _ALIGN_PAIRS for o in pair)
|
||||
# (NUM_TILES, COPY_BLOCK_SIZE) configs. NUM_TILES=1 is the SD conv callsite
|
||||
# (single-CTA memcpy) where tile partitioning collapses, so COPY_BLOCK_SIZE
|
||||
# is not observable — one value suffices. NUM_TILES=_TEMPORAL_TILES is the
|
||||
# temporal callsite: 1024 matches the production launch; 8 shrinks per-tile
|
||||
# rounding so ``_COPY_SIZES``' 1024/4096 cases cross tile boundaries
|
||||
# end-to-end (partitioning math is COPY_BLOCK_SIZE-agnostic, so a small
|
||||
# value proves it for all values).
|
||||
_TILE_BLOCK_CONFIGS = [
|
||||
(1, 1024),
|
||||
(_TEMPORAL_TILES, 1024),
|
||||
(_TEMPORAL_TILES, 8),
|
||||
]
|
||||
|
||||
|
||||
@_parametrize("copy_size", _COPY_SIZES)
|
||||
@_parametrize("src_off,dst_off", _ALIGN_PAIRS)
|
||||
@_parametrize("num_tiles,copy_block_size", _TILE_BLOCK_CONFIGS)
|
||||
def test_memcpy_u64_tiled_matches_slice_copy(
|
||||
copy_size, src_off, dst_off, num_tiles, copy_block_size
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Pad by the max possible offset so the reference slice is in-bounds.
|
||||
slack = _MAX_ALIGN_OFF + 8
|
||||
src = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
|
||||
# Start dst from a distinct random pattern so the "unchanged region"
|
||||
# check catches any accidental out-of-range writes.
|
||||
dst = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
|
||||
ref = dst.clone()
|
||||
ref[dst_off : dst_off + copy_size] = src[src_off : src_off + copy_size]
|
||||
|
||||
_memcpy_wrapper_kernel[(num_tiles,)](
|
||||
src,
|
||||
dst,
|
||||
copy_size,
|
||||
src_off,
|
||||
dst_off,
|
||||
COPY_BLOCK_SIZE=copy_block_size,
|
||||
NUM_TILES=num_tiles,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Byte-identical: the memcpy is dtype-agnostic; any diff is a bug.
|
||||
torch.testing.assert_close(dst, ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for cs in _COPY_SIZES:
|
||||
for so, do in _ALIGN_PAIRS:
|
||||
for nt, cbs in _TILE_BLOCK_CONFIGS:
|
||||
test_memcpy_u64_tiled_matches_slice_copy(cs, so, do, nt, cbs)
|
||||
print("ok")
|
||||
@@ -1,512 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
|
||||
|
||||
The fused "align" pre-copy must migrate mamba state across block boundaries
|
||||
with byte-identical semantics to the scalar V1 copy specs
|
||||
(``get_conv_copy_spec`` / ``get_temporal_copy_spec``):
|
||||
|
||||
* conv state (SD layout, conv_width > 0): shift the sliding window by
|
||||
``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
|
||||
``state[bt[dst_col], :conv_width - token_bias]``.
|
||||
* temporal state (conv_width == 0): ``token_bias`` selects the accepted
|
||||
speculative column -- ``state[bt[src_col + token_bias]]`` ->
|
||||
``state[bt[dst_col]]``.
|
||||
|
||||
The kernel must also no-op when ``src_col < 0`` (fresh request) or
|
||||
``src_col == dst_col`` (no boundary crossed). V2 callers pass an explicit
|
||||
``idx_mapping``; V1 align preprocessing launches in batch order with
|
||||
``idx_mapping=None``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker import mamba_utils as worker_mamba_utils
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
_TEMPORAL_TILES,
|
||||
postprocess_mamba_fused_kernel,
|
||||
precopy_mamba_align_fused_kernel,
|
||||
)
|
||||
|
||||
_parametrize: Callable[..., Callable[[Any], Any]]
|
||||
|
||||
try:
|
||||
import pytest
|
||||
|
||||
_cuda_required = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
|
||||
)
|
||||
_parametrize = pytest.mark.parametrize
|
||||
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
|
||||
|
||||
def _cuda_required(fn):
|
||||
return fn
|
||||
|
||||
def _no_parametrize(_name, _values):
|
||||
def _deco(fn):
|
||||
return fn
|
||||
|
||||
return _deco
|
||||
|
||||
_parametrize = _no_parametrize
|
||||
|
||||
|
||||
NUM_LAYERS = 3
|
||||
CONV_WIDTH = 4 # conv_kernel - 1 + num_spec
|
||||
CONV_DIM = 96
|
||||
SSM_SHAPE = (4, 16, 16)
|
||||
MAX_COLS = 8
|
||||
|
||||
|
||||
def _build_state(num_blocks, device, conv_state_dim_first):
|
||||
"""Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
|
||||
convs, ssms = [], []
|
||||
for _ in range(NUM_LAYERS):
|
||||
conv_shape = (
|
||||
(num_blocks, CONV_DIM, CONV_WIDTH)
|
||||
if conv_state_dim_first
|
||||
else (num_blocks, CONV_WIDTH, CONV_DIM)
|
||||
)
|
||||
convs.append(
|
||||
torch.randn(
|
||||
*conv_shape,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
ssms.append(
|
||||
torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
|
||||
)
|
||||
return convs, ssms
|
||||
|
||||
|
||||
def _build_meta(convs, ssms, device, conv_state_dim_first):
|
||||
"""Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
|
||||
n = NUM_LAYERS * 2
|
||||
base = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
elem = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
inner = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
width = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
group = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD)
|
||||
drs = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
i = 0
|
||||
for layer in range(NUM_LAYERS):
|
||||
conv, ssm = convs[layer], ssms[layer]
|
||||
# conv (SD): width = size(1), inner = stride(1)
|
||||
base[i] = conv.data_ptr()
|
||||
blk_stride[i] = conv.stride(0) * conv.element_size()
|
||||
elem[i] = conv.element_size()
|
||||
if conv_state_dim_first:
|
||||
width[i] = conv.size(2)
|
||||
inner[i] = 1
|
||||
drc[i] = conv.size(1)
|
||||
drs[i] = conv.stride(1) * conv.element_size()
|
||||
else:
|
||||
width[i] = conv.size(1)
|
||||
inner[i] = conv.stride(1)
|
||||
i += 1
|
||||
# ssm (temporal): width = 0, inner = elems per block
|
||||
base[i] = ssm.data_ptr()
|
||||
blk_stride[i] = ssm.stride(0) * ssm.element_size()
|
||||
elem[i] = ssm.element_size()
|
||||
width[i] = 0
|
||||
inner[i] = ssm[0].numel()
|
||||
i += 1
|
||||
return base, blk_stride, elem, inner, width, group, drc, drs
|
||||
|
||||
|
||||
def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs, conv_dim_first):
|
||||
"""Apply the V1 copy semantics on clones, reading from the pre-copy state."""
|
||||
conv_pre = [c.clone() for c in convs]
|
||||
ssm_pre = [s.clone() for s in ssms]
|
||||
conv_ref = [c.clone() for c in convs]
|
||||
ssm_ref = [s.clone() for s in ssms]
|
||||
for r in range(num_reqs):
|
||||
sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
|
||||
if sc < 0 or sc == dc:
|
||||
continue
|
||||
sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
|
||||
tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias
|
||||
for layer in range(NUM_LAYERS):
|
||||
if conv_dim_first:
|
||||
conv_ref[layer][dblk, :, : CONV_WIDTH - tb] = conv_pre[layer][
|
||||
sblk, :, tb:
|
||||
]
|
||||
else:
|
||||
conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
|
||||
ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
|
||||
return conv_ref, ssm_ref
|
||||
|
||||
|
||||
@_parametrize("conv_state_dim_first", [False, True])
|
||||
@_parametrize("num_reqs", [1, 4, 16])
|
||||
@_parametrize("token_bias", [0, 1, 2])
|
||||
@_parametrize("has_idx_mapping", [True, False])
|
||||
@_parametrize("temporal_tiles", [1, _TEMPORAL_TILES])
|
||||
@_cuda_required
|
||||
def test_precopy_matches_v1_copy_specs(
|
||||
num_reqs,
|
||||
token_bias,
|
||||
has_idx_mapping,
|
||||
conv_state_dim_first,
|
||||
temporal_tiles,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
# Distinct physical block per (req, col) so copies never alias.
|
||||
num_blocks = num_reqs * MAX_COLS + 1
|
||||
bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
|
||||
for r in range(num_reqs):
|
||||
bt[r] = torch.arange(
|
||||
1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
|
||||
# the rest cross from col 1 -> col 0 with the given spec token bias.
|
||||
src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
|
||||
dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
|
||||
if num_reqs >= 1:
|
||||
src_col[0] = -1 # fresh -> no copy
|
||||
if num_reqs >= 2:
|
||||
dst_col[1] = 1 # src_col == dst_col -> no copy
|
||||
|
||||
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
|
||||
conv_ref, ssm_ref = _reference(
|
||||
convs,
|
||||
ssms,
|
||||
bt.cpu(),
|
||||
src_col.cpu(),
|
||||
dst_col.cpu(),
|
||||
bias.cpu(),
|
||||
num_reqs,
|
||||
conv_state_dim_first,
|
||||
)
|
||||
|
||||
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
|
||||
convs, ssms, device, conv_state_dim_first
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
# Non-identity mapping: block-table rows are request slots, not batch rows.
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device).flip(0)
|
||||
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
dst_col,
|
||||
src_col,
|
||||
bias,
|
||||
bt_ptrs,
|
||||
bt.stride(0),
|
||||
base,
|
||||
blk_stride,
|
||||
elem,
|
||||
inner,
|
||||
width,
|
||||
group,
|
||||
drc,
|
||||
drs,
|
||||
idx_mapping if has_idx_mapping else None,
|
||||
num_reqs,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=conv_state_dim_first,
|
||||
HAS_IDX_MAPPING=has_idx_mapping,
|
||||
TEMPORAL_TILES=temporal_tiles,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for layer in range(NUM_LAYERS):
|
||||
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
|
||||
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
|
||||
|
||||
|
||||
@_parametrize("conv_state_dim_first", [False, True])
|
||||
@_cuda_required
|
||||
def test_v2_postprocess_indexes_block_tables_by_req_slot(conv_state_dim_first):
|
||||
"""Under PP the V2 postprocess runs after later steps have re-gathered the
|
||||
batch-ordered tables, so it must index the request-slot tables by slot."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
num_reqs = 4
|
||||
block_size = 8
|
||||
num_blocks = num_reqs * MAX_COLS + 1
|
||||
bt = torch.arange(1, num_blocks, dtype=torch.int32, device=device).view(
|
||||
num_reqs, MAX_COLS
|
||||
)
|
||||
# slot 0: copy col 1 -> 0; slot 1: src == dst; slot 2: not aligned;
|
||||
# slot 3: copy col 2 -> 0 with one accepted draft token.
|
||||
num_accepted = torch.tensor([1, 1, 1, 2], dtype=torch.int32, device=device)
|
||||
state_idx = torch.tensor([1, 0, 1, 2], dtype=torch.int32, device=device)
|
||||
new_num_computed = torch.tensor([8, 8, 5, 8], dtype=torch.int32, device=device)
|
||||
ref_src = torch.tensor([1, 0, -1, 2], dtype=torch.int32)
|
||||
ref_dst = torch.zeros(num_reqs, dtype=torch.int32)
|
||||
ref_bias = torch.tensor([0, 0, 0, 1], dtype=torch.int32)
|
||||
|
||||
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
|
||||
conv_ref, ssm_ref = _reference(
|
||||
convs,
|
||||
ssms,
|
||||
bt.cpu(),
|
||||
ref_src,
|
||||
ref_dst,
|
||||
ref_bias,
|
||||
num_reqs,
|
||||
conv_state_dim_first,
|
||||
)
|
||||
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
|
||||
convs, ssms, device, conv_state_dim_first
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
idx_mapping = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device=device)
|
||||
postprocess_mamba_fused_kernel[(num_reqs, NUM_LAYERS * 2, 1)](
|
||||
num_accepted.clone(),
|
||||
state_idx,
|
||||
None,
|
||||
new_num_computed,
|
||||
None,
|
||||
bt_ptrs,
|
||||
bt.stride(0),
|
||||
base,
|
||||
blk_stride,
|
||||
elem,
|
||||
inner,
|
||||
width,
|
||||
group,
|
||||
drc,
|
||||
drs,
|
||||
num_accepted,
|
||||
idx_mapping,
|
||||
num_reqs,
|
||||
block_size=block_size,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=conv_state_dim_first,
|
||||
HAS_IDX_MAPPING=True,
|
||||
PRECOMPUTED_NEW_COMPUTED=True,
|
||||
TEMPORAL_TILES=1,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for layer in range(NUM_LAYERS):
|
||||
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
|
||||
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ds_conv_copy_spec_reproduces_multi_accept_assert(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
layer_mamba_utils,
|
||||
"is_conv_state_dim_first",
|
||||
lambda: True,
|
||||
)
|
||||
state = torch.empty((2, CONV_DIM, CONV_WIDTH), dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(AssertionError, match="num_accepted_tokens > 1"):
|
||||
layer_mamba_utils.get_conv_copy_spec(
|
||||
state=state,
|
||||
block_ids=[0, 1],
|
||||
cur_block_idx=0,
|
||||
num_accepted_tokens=3,
|
||||
)
|
||||
|
||||
|
||||
class _FakeCpuGpuBuffer:
|
||||
def __init__(self, n):
|
||||
self.np = np.zeros(n, dtype=np.int32)
|
||||
self.gpu = object()
|
||||
self.copy_sizes = []
|
||||
|
||||
def copy_to_gpu(self, n=None):
|
||||
self.copy_sizes.append(n)
|
||||
return self.gpu
|
||||
|
||||
|
||||
class _FakePrecopyContext:
|
||||
def __init__(self, n):
|
||||
self.is_initialized = True
|
||||
self.mamba_group_ids = [0]
|
||||
self.mamba_state_idx_buf = _FakeCpuGpuBuffer(n)
|
||||
self.precopy_src_col_buf = _FakeCpuGpuBuffer(n)
|
||||
self.precopy_token_bias_buf = _FakeCpuGpuBuffer(n)
|
||||
self.calls = []
|
||||
|
||||
def initialize_from_forward_context(self, *args, **kwargs):
|
||||
raise AssertionError("test context is pre-initialized")
|
||||
|
||||
def run_fused_precopy(
|
||||
self,
|
||||
*,
|
||||
num_reqs,
|
||||
state_idx_gpu,
|
||||
src_col_gpu,
|
||||
token_bias_gpu,
|
||||
idx_mapping,
|
||||
):
|
||||
self.calls.append(
|
||||
{
|
||||
"num_reqs": num_reqs,
|
||||
"state_idx": self.mamba_state_idx_buf.np[:num_reqs].copy(),
|
||||
"src_col": self.precopy_src_col_buf.np[:num_reqs].copy(),
|
||||
"token_bias": self.precopy_token_bias_buf.np[:num_reqs].copy(),
|
||||
"idx_mapping": idx_mapping,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_preprocess_case(token_bias):
|
||||
req_ids = ["fresh", "same", "cross_a", "cross_b"]
|
||||
scheduler_output = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_cached_reqs=SimpleNamespace(resumed_req_ids=set()),
|
||||
num_scheduled_tokens={
|
||||
"fresh": 1,
|
||||
"same": 1,
|
||||
"cross_a": 1,
|
||||
"cross_b": 2,
|
||||
},
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
req_ids=req_ids,
|
||||
num_accepted_tokens_cpu=np.array(
|
||||
[token_bias + 1, token_bias + 1, token_bias + 1, 2],
|
||||
dtype=np.int32,
|
||||
),
|
||||
)
|
||||
requests = {
|
||||
"fresh": SimpleNamespace(req_id="fresh", num_computed_tokens=0),
|
||||
"same": SimpleNamespace(req_id="same", num_computed_tokens=5),
|
||||
"cross_a": SimpleNamespace(req_id="cross_a", num_computed_tokens=8),
|
||||
"cross_b": SimpleNamespace(req_id="cross_b", num_computed_tokens=7),
|
||||
}
|
||||
mamba_state_idx = {"same": 1, "cross_a": 0, "cross_b": 1}
|
||||
return scheduler_output, input_batch, requests, mamba_state_idx
|
||||
|
||||
|
||||
@_parametrize("token_bias", [1, 2])
|
||||
def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bias):
|
||||
block_size = 4
|
||||
mamba_spec = SimpleNamespace(block_size=block_size, num_speculative_blocks=1)
|
||||
# Fakes stand in for real config/context objects. The fused path tests
|
||||
# below only touch the attributes preprocess_mamba actually reads.
|
||||
cache_config: Any = SimpleNamespace(enable_prefix_caching=True)
|
||||
kv_cache_config: Any = SimpleNamespace()
|
||||
scalar_copy_calls = []
|
||||
|
||||
def fake_collect(
|
||||
copy_bufs,
|
||||
kv_cache_config,
|
||||
mamba_state_copy_funcs,
|
||||
mamba_group_ids,
|
||||
src_block_idx,
|
||||
dest_block_idx,
|
||||
accept_token_bias,
|
||||
req_state,
|
||||
forward_context,
|
||||
):
|
||||
scalar_copy_calls.append(
|
||||
(
|
||||
req_state.req_id,
|
||||
src_block_idx,
|
||||
dest_block_idx,
|
||||
accept_token_bias,
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(worker_mamba_utils, "collect_mamba_copy_meta", fake_collect)
|
||||
monkeypatch.setattr(
|
||||
worker_mamba_utils, "do_mamba_copy_block", lambda copy_bufs: None
|
||||
)
|
||||
|
||||
scalar_case = _make_preprocess_case(token_bias)
|
||||
fused_case = _make_preprocess_case(token_bias)
|
||||
|
||||
scalar_copy_bufs: Any = SimpleNamespace(
|
||||
mamba_group_ids=[0],
|
||||
mamba_spec=mamba_spec,
|
||||
offset=0,
|
||||
)
|
||||
worker_mamba_utils.preprocess_mamba(
|
||||
scheduler_output=scalar_case[0],
|
||||
kv_cache_config=kv_cache_config,
|
||||
cache_config=cache_config,
|
||||
mamba_state_idx=scalar_case[3],
|
||||
input_batch=scalar_case[1],
|
||||
requests=scalar_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=scalar_copy_bufs,
|
||||
)
|
||||
|
||||
ctx: Any = _FakePrecopyContext(len(fused_case[1].req_ids))
|
||||
fused_copy_bufs: Any = SimpleNamespace(
|
||||
mamba_group_ids=[0],
|
||||
mamba_spec=mamba_spec,
|
||||
offset=0,
|
||||
)
|
||||
worker_mamba_utils.preprocess_mamba(
|
||||
scheduler_output=fused_case[0],
|
||||
kv_cache_config=kv_cache_config,
|
||||
cache_config=cache_config,
|
||||
mamba_state_idx=fused_case[3],
|
||||
input_batch=fused_case[1],
|
||||
requests=fused_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=fused_copy_bufs,
|
||||
align_ctx=ctx,
|
||||
)
|
||||
|
||||
assert fused_case[3] == scalar_case[3]
|
||||
np.testing.assert_array_equal(
|
||||
fused_case[1].num_accepted_tokens_cpu,
|
||||
scalar_case[1].num_accepted_tokens_cpu,
|
||||
)
|
||||
assert scalar_copy_calls == [
|
||||
("cross_a", 0, 2, token_bias),
|
||||
("cross_b", 1, 2, 1),
|
||||
]
|
||||
assert len(ctx.calls) == 1
|
||||
call = ctx.calls[0]
|
||||
assert call["num_reqs"] == len(fused_case[1].req_ids)
|
||||
assert call["idx_mapping"] is None
|
||||
np.testing.assert_array_equal(call["state_idx"], np.array([0, 1, 2, 2]))
|
||||
np.testing.assert_array_equal(call["src_col"], np.array([-1, -1, 0, 1]))
|
||||
np.testing.assert_array_equal(call["token_bias"], np.array([0, 0, token_bias, 1]))
|
||||
fused_copy_calls = [
|
||||
(req_id, int(src), int(dst), int(bias))
|
||||
for req_id, src, dst, bias in zip(
|
||||
fused_case[1].req_ids,
|
||||
call["src_col"],
|
||||
call["state_idx"],
|
||||
call["token_bias"],
|
||||
)
|
||||
if int(src) != -1 and int(src) != int(dst)
|
||||
]
|
||||
assert fused_copy_calls == scalar_copy_calls
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from itertools import product
|
||||
|
||||
_CASES = product(
|
||||
(1, 4, 16), # num_reqs
|
||||
(0, 1, 2), # token_bias
|
||||
(True, False), # has_idx_mapping
|
||||
(False, True), # conv_state_dim_first
|
||||
(1, _TEMPORAL_TILES), # temporal_tiles
|
||||
)
|
||||
for nr, tb, mapping, dim_first, tt in _CASES:
|
||||
test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first, tt)
|
||||
print(
|
||||
f"OK num_reqs={nr} token_bias={tb} has_idx_mapping={mapping} "
|
||||
f"conv_dim_first={dim_first} temporal_tiles={tt}"
|
||||
)
|
||||
@@ -1,248 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prefill == decode equivalence for the Mamba2 ReplaySSM kernels.
|
||||
|
||||
The SSM recurrence is path-independent: vLLM's chunked prefill (the SSD kernel
|
||||
``mamba_chunk_scan_combined_varlen``) and step-by-step decode must produce the
|
||||
same per-position outputs and final state over a sequence. This file feeds one
|
||||
set of inputs through the production dt flow (raw dt + softplus + a per-head
|
||||
dt_bias, applied inside each kernel) to:
|
||||
|
||||
* the exact fp32 step recurrence (``selective_state_update_ref``) -- the
|
||||
ground truth,
|
||||
* the chunked prefill kernel,
|
||||
* the baseline decode kernel,
|
||||
* the ReplaySSM output_only decode kernel,
|
||||
|
||||
and checks all of them agree. Prefill (a chunked scan) and decode (a step
|
||||
recurrence) are different code paths, so they differ numerically: the chunked
|
||||
scan carries ~2e-2 (fp32) / ~4e-2 (bf16) vs the exact recurrence, far above the
|
||||
near-exact decode. We therefore anchor every path on the exact recurrence at
|
||||
SSD-level tolerances (the same regime as ``test_mamba_ssm_ssd.py``), which the
|
||||
chunked scan sets, keyed off the activation dtype.
|
||||
|
||||
State and activation/buffer precision are swept independently, including the
|
||||
fp32-state + bf16-activation production config.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.utils import selective_state_update_ref
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import compute_varlen_chunk_metadata
|
||||
|
||||
|
||||
def _prefill_tolerances(act_dtype: torch.dtype) -> tuple[float, float]:
|
||||
# The chunked prefill scan, not the decode, sets these: it carries ~2e-2
|
||||
# (fp32) / ~6e-2 (bf16) vs the exact recurrence, while ReplaySSM decode is
|
||||
# near-exact (~2e-6 fp32). Keyed off the activation dtype (the outputs are
|
||||
# in act_dtype). Same regime as test_mamba_ssm_ssd.py.
|
||||
if act_dtype == torch.float32:
|
||||
return 1e-2, 3e-2
|
||||
return 6e-2, 1e-1
|
||||
|
||||
|
||||
def _run_prefill_decode_equivalence(
|
||||
*,
|
||||
state_dtype: torch.dtype,
|
||||
act_dtype: torch.dtype,
|
||||
nheads: int,
|
||||
headdim: int,
|
||||
ngroups: int,
|
||||
dstate: int,
|
||||
seqlen: int,
|
||||
chunk_size: int,
|
||||
max_cache_len: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
"""Prefill the whole sequence and decode it step by step; check both match
|
||||
the exact fp32 recurrence (and each other). All paths use the production dt
|
||||
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
|
||||
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
|
||||
device = current_platform.device_type
|
||||
rtol, atol = _prefill_tolerances(act_dtype)
|
||||
set_random_seed(seed)
|
||||
|
||||
# Production dt flow: raw dt + a per-head dt_bias (~-4 keeps softplus(dt +
|
||||
# bias) small and well-conditioned). dt_bias is (nheads,) for prefill and
|
||||
# (nheads, headdim) for the decode kernels/reference.
|
||||
A = -torch.exp(torch.rand(nheads, device=device, dtype=act_dtype))
|
||||
dt = torch.randn(seqlen, nheads, device=device, dtype=act_dtype)
|
||||
dt_bias = torch.rand(nheads, device=device, dtype=act_dtype) - 4
|
||||
dt_bias_hd = dt_bias.view(nheads, 1).expand(nheads, headdim)
|
||||
X = torch.randn(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
B = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
C = torch.randn(seqlen, ngroups, dstate, device=device, dtype=act_dtype)
|
||||
A_bcast = A.view(nheads, 1, 1).expand(nheads, headdim, dstate)
|
||||
|
||||
# Chunked prefill over the whole sequence (implicit batch=1, varlen). The
|
||||
# kernel always returns the final state in fp32, so no state_dtype plumbing.
|
||||
cu_seqlens = torch.tensor((0, seqlen), device=device).cumsum(0).to(torch.int32)
|
||||
cu_chunk_seqlens, last_chunk_indices, seq_idx = compute_varlen_chunk_metadata(
|
||||
cu_seqlens, chunk_size
|
||||
)
|
||||
y_prefill = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
final_state_prefill = mamba_chunk_scan_combined_varlen(
|
||||
X,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_chunk_seqlens=cu_chunk_seqlens,
|
||||
last_chunk_indices=last_chunk_indices,
|
||||
seq_idx=seq_idx,
|
||||
out=y_prefill,
|
||||
D=None,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
)
|
||||
|
||||
# Step paths: exact fp32 recurrence (ground truth), baseline, ReplaySSM.
|
||||
# State follows state_dtype; caches follow act_dtype (dt_cache is fp32).
|
||||
state_ref = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=torch.float32
|
||||
)
|
||||
state_base = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=state_dtype
|
||||
)
|
||||
state_dec = torch.zeros(
|
||||
1, nheads, headdim, dstate, device=device, dtype=state_dtype
|
||||
)
|
||||
x_cache = torch.zeros(
|
||||
1, nheads, max_cache_len, headdim, device=device, dtype=act_dtype
|
||||
)
|
||||
dt_cache = torch.zeros(1, nheads, max_cache_len, device=device, dtype=torch.float32)
|
||||
B_cache = torch.zeros(
|
||||
1, ngroups, max_cache_len, dstate, device=device, dtype=act_dtype
|
||||
)
|
||||
bc_pre = torch.empty(1, ngroups, max_cache_len, device=device, dtype=torch.float32)
|
||||
write_pos = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
# No skip connection (D=0) on any path here; the D!=0 path is covered by the
|
||||
# standard-decode suite. The baseline kernel needs a D tensor, not None.
|
||||
D_zero = torch.zeros(nheads, headdim, device=device)
|
||||
|
||||
y_ref = torch.empty(seqlen, nheads, headdim, device=device, dtype=torch.float32)
|
||||
y_base = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
y_dec = torch.empty(seqlen, nheads, headdim, device=device, dtype=act_dtype)
|
||||
for t in range(seqlen):
|
||||
dt_t = dt[t].view(1, nheads, 1).expand(1, nheads, headdim)
|
||||
is_flush = write_pos == max_cache_len - 1
|
||||
|
||||
y_ref[t] = selective_state_update_ref(
|
||||
state_ref,
|
||||
X[t : t + 1].float(),
|
||||
dt_t.float(),
|
||||
A_bcast.float(),
|
||||
B[t : t + 1].float(),
|
||||
C[t : t + 1].float(),
|
||||
dt_bias=dt_bias_hd.float(),
|
||||
dt_softplus=True,
|
||||
)[0]
|
||||
|
||||
out_b = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
|
||||
selective_state_update(
|
||||
state_base,
|
||||
X[t : t + 1],
|
||||
dt_t,
|
||||
A_bcast,
|
||||
B[t : t + 1],
|
||||
C[t : t + 1],
|
||||
D=D_zero,
|
||||
dt_bias=dt_bias_hd,
|
||||
dt_softplus=True,
|
||||
out=out_b,
|
||||
)
|
||||
y_base[t] = out_b[0]
|
||||
|
||||
out_d = torch.empty(1, nheads, headdim, device=device, dtype=act_dtype)
|
||||
common = dict(
|
||||
dt_bias=dt_bias_hd,
|
||||
dt_softplus=True,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
write_pos=write_pos,
|
||||
is_flush=is_flush,
|
||||
max_cache_len=max_cache_len,
|
||||
out=out_d,
|
||||
)
|
||||
selective_state_update_replayssm_output_only(
|
||||
state_dec,
|
||||
X[t : t + 1],
|
||||
dt_t,
|
||||
A_bcast,
|
||||
B[t : t + 1],
|
||||
C[t : t + 1],
|
||||
bc_pre=bc_pre,
|
||||
**common,
|
||||
)
|
||||
y_dec[t] = out_d[0]
|
||||
|
||||
write_pos = torch.where(is_flush, torch.zeros_like(write_pos), write_pos + 1)
|
||||
|
||||
# Every path computes the same recurrence; anchor each on the fp32 truth.
|
||||
torch.testing.assert_close(y_prefill.float(), y_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(y_base.float(), y_ref, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(y_dec.float(), y_ref, rtol=rtol, atol=atol)
|
||||
# Headline: ReplaySSM decode matches the chunked prefill directly.
|
||||
torch.testing.assert_close(y_dec.float(), y_prefill.float(), rtol=rtol, atol=atol)
|
||||
# Final state too (the recurrence ends in state_ref after the loop).
|
||||
torch.testing.assert_close(
|
||||
final_state_prefill[0].float(), state_ref[0], rtol=rtol, atol=atol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Mamba2 ReplaySSM kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"precision",
|
||||
# fp32 state is the default; bf16/fp16 are reduced-footprint configs. fp16
|
||||
# appears as an activation dtype (s32_afp16, sfp16_afp16) and as a finer-
|
||||
# mantissa state under bf16 activations (sfp16_a16).
|
||||
[
|
||||
pytest.param((torch.float32, torch.float32), id="s32_a32"),
|
||||
pytest.param((torch.float32, torch.bfloat16), id="s32_a16"),
|
||||
pytest.param((torch.bfloat16, torch.bfloat16), id="s16_a16"),
|
||||
pytest.param((torch.float32, torch.float16), id="s32_afp16"),
|
||||
pytest.param((torch.float16, torch.float16), id="sfp16_afp16"),
|
||||
pytest.param((torch.float16, torch.bfloat16), id="sfp16_a16"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"geometry", # (nheads, headdim, dstate, ngroups)
|
||||
[
|
||||
pytest.param((8, 64, 64, 2), id="small"),
|
||||
pytest.param((96, 80, 128, 8), id="nano4b"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_prefill_decode_equivalence(
|
||||
precision: tuple[torch.dtype, torch.dtype],
|
||||
geometry: tuple[int, int, int, int],
|
||||
):
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
_run_prefill_decode_equivalence(
|
||||
state_dtype=state_dtype,
|
||||
act_dtype=act_dtype,
|
||||
nheads=nheads,
|
||||
headdim=headdim,
|
||||
ngroups=ngroups,
|
||||
dstate=dstate,
|
||||
seqlen=16,
|
||||
chunk_size=8,
|
||||
max_cache_len=4,
|
||||
)
|
||||
Reference in New Issue
Block a user