[CPU][Spec Decode] Optimize GDN conv path for speculative decoding (#48577)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-07-27 06:20:14 +00:00
committed by GitHub
co-authored by Codex Li, Jiang <jiang1.li@intel.com>
parent c314af1abf
commit 544cb724c8
5 changed files with 767 additions and 122 deletions
+435 -1
View File
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import types
import pytest
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from vllm.model_executor.layers.mamba.ops.cpu import gdn_attention
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -417,6 +419,207 @@ def _conv_inputs(total_tokens: int):
return x, weight, bias
def _sd_conv_states(
num_slots: int, state_len: int, dim: int = CONV_DIM
) -> torch.Tensor:
storage = torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16)
return storage.transpose(1, 2)
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
@torch.inference_mode()
def test_spec_aware_mixed_routing_preserves_token_order(
monkeypatch: pytest.MonkeyPatch,
) -> None:
num_tokens = 4
projection = torch.arange(num_tokens * 16, dtype=torch.float32).view(num_tokens, 16)
mixed_qkv, b, a = projection[:, :4], projection[:, 4:6], projection[:, 6:8]
assert all(not tensor.is_contiguous() for tensor in (mixed_qkv, b, a))
spec_indices = torch.tensor([0, 2])
nonspec_indices = torch.tensor([1, 3])
metadata = types.SimpleNamespace(
spec_sequence_masks=torch.ones(1, dtype=torch.bool),
spec_token_indx=spec_indices,
non_spec_token_indx=nonspec_indices,
num_prefills=1,
num_decodes=0,
)
routed = []
def record(*args):
routed.append(args[2:5])
return args[2]
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
monkeypatch.setattr(gdn_attention, "_spec_forward", record)
monkeypatch.setattr(gdn_attention, "_spec_aware_nonspec_subset", record)
layer = types.SimpleNamespace(
kv_cache=[torch.empty(1, 4, 6), torch.empty(1, 1, 1, 1)]
)
core_attn_out = torch.empty_like(mixed_qkv)
gdn_attention._cpu_gdn_attention_spec_aware(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv=mixed_qkv,
b=b,
a=a,
core_attn_out=core_attn_out,
width=CONV_KERNEL,
state_len=6,
)
expected_inputs = (mixed_qkv, b, a)
assert len(routed) == 2
for actual_inputs, indices in zip(routed, (spec_indices, nonspec_indices)):
for actual, expected in zip(actual_inputs, expected_inputs):
assert actual.is_contiguous()
torch.testing.assert_close(actual, expected.index_select(0, indices))
torch.testing.assert_close(core_attn_out, mixed_qkv)
@torch.inference_mode()
def test_spec_aware_nonspec_materializes_state_indices(
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.arange(8, dtype=torch.int32).view(2, 4)
state_indices = block_table[:, 0]
assert not state_indices.is_contiguous()
metadata = types.SimpleNamespace(
non_spec_state_indices_tensor=state_indices,
non_spec_query_start_loc=torch.tensor([0, 2, 4], dtype=torch.int32),
num_decodes=0,
num_decode_tokens=0,
num_prefills=2,
num_prefill_tokens=4,
has_initial_state=torch.tensor([False, False]),
)
recorded_indices = None
def causal_conv1d_fwd_cpu(**kwargs):
nonlocal recorded_indices
recorded_indices = kwargs["cache_indices"]
return kwargs["x"]
def fused_gdn_gating_cpu(**kwargs):
return kwargs["a"], kwargs["b"]
def chunk_gated_delta_rule_cpu(**kwargs):
out = torch.zeros(1, 4, 1, 1)
return out, kwargs["initial_state"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_fwd_cpu", causal_conv1d_fwd_cpu
)
monkeypatch.setattr(gdn_attention.ops, "fused_gdn_gating_cpu", fused_gdn_gating_cpu)
monkeypatch.setattr(
gdn_attention.ops,
"chunk_gated_delta_rule_cpu",
chunk_gated_delta_rule_cpu,
)
layer = types.SimpleNamespace(
activation="silu",
conv1d=types.SimpleNamespace(weight=torch.empty(0), bias=None),
A_log=torch.empty(0),
dt_bias=torch.empty(0),
rearrange_mixed_qkv=lambda x: (
x[:, :1].view(1, 4, 1, 1),
x[:, :1].view(1, 4, 1, 1),
x[:, :1].view(1, 4, 1, 1),
),
)
gdn_attention._spec_aware_nonspec(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv=torch.zeros(4, 4),
b=torch.zeros(4, 1),
a=torch.zeros(4, 1),
core_attn_out=torch.zeros(4, 1, 1),
conv_buf=torch.zeros(8, 1, 6),
ssm_state=torch.zeros(8, 1, 1, 1),
width=4,
)
assert recorded_indices is not None
assert recorded_indices.is_contiguous()
torch.testing.assert_close(
recorded_indices, torch.tensor([0, 4], dtype=torch.int32)
)
@torch.inference_mode()
def test_spec_forward_prepares_native_conv_metadata(
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
state_indices = block_table[:, 0]
accepted_counts = torch.tensor([1, 4], dtype=torch.int32)
assert not state_indices.is_contiguous()
metadata = types.SimpleNamespace(
num_spec_decodes=2,
spec_state_indices_tensor=block_table,
spec_query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32),
num_accepted_tokens=accepted_counts,
)
forwarded_indices = None
forwarded_counts = None
def causal_conv1d_update_cpu(**kwargs):
nonlocal forwarded_counts, forwarded_indices
forwarded_indices = kwargs["conv_state_indices"]
forwarded_counts = kwargs["num_accepted_tokens"]
return kwargs["x"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
)
monkeypatch.setattr(
gdn_attention.ops,
"fused_sigmoid_gating_delta_rule_update_spec_cpu",
lambda **kwargs: kwargs["q"],
)
layer = types.SimpleNamespace(
activation="silu",
conv1d=types.SimpleNamespace(weight=torch.empty(1, CONV_KERNEL), bias=None),
A_log=None,
dt_bias=None,
rearrange_mixed_qkv=lambda x: (x.unsqueeze(0),) * 3,
)
gdn_attention._spec_forward(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv_spec=torch.zeros(8, 1, dtype=torch.bfloat16),
b_spec=torch.empty(0),
a_spec=torch.empty(0),
conv_buf=torch.empty(0),
ssm_state=torch.empty(0),
width=CONV_KERNEL,
state_len=0,
)
expected = (
(forwarded_indices, torch.tensor([0, 4], dtype=torch.int32)),
(forwarded_counts, torch.tensor([1, 4], dtype=torch.int32)),
)
for actual, reference in expected:
assert actual is not None
assert actual.is_contiguous()
assert actual.dtype == torch.int32
torch.testing.assert_close(actual, reference)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> None:
@@ -475,7 +678,182 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="causal_conv1d_fwd_cpu requires AMX/AVX512",
reason="requires AMX support",
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
batch_size = 3
is_vnni = True
x, weight, bias = _conv_inputs(batch_size)
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
narrow_state = _sd_conv_states(batch_size, state_len)
narrow_state.copy_(
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
)
wide_state = _sd_conv_states(batch_size, wide_state_len)
wide_state[:, :, :state_len].copy_(narrow_state)
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out_narrow = ops.causal_conv1d_update_cpu(
x=x,
conv_states=narrow_state,
weight=conv_weight,
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
)
out_wide = ops.causal_conv1d_update_cpu(
x=x,
conv_states=wide_state,
weight=conv_weight,
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
)
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
def _ref_causal_conv1d_update_cpu_multi(
x: torch.Tensor,
conv_states: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
) -> torch.Tensor:
batch_size, seq_len, dim = x.shape
state_len = conv_states.size(2)
conv_out = torch.empty_like(x)
conv_weight = weight.unsqueeze(1)
for i in range(batch_size):
slot = int(conv_state_indices[i].item())
offset = int(num_accepted_tokens[i].item()) - 1
state = conv_states[slot]
x_seq = x[i].transpose(0, 1).to(state.dtype)
prior = state[:, offset : offset + CONV_KERNEL - 1]
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0)
out = F.conv1d(conv_in, conv_weight, bias, groups=dim)[0]
if silu_activation:
out = F.silu(out)
conv_out[i] = out.transpose(0, 1).to(conv_out.dtype)
keep = state[:, offset + 1 : offset + 1 + (state_len - seq_len)]
state.copy_(torch.cat([keep, x_seq], dim=-1))
return conv_out
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize(
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
[
(1, 1, [1], False, False, False),
(1, 1, [1], True, True, True),
(4, 4, [1, 2, 3, 4], False, True, False),
(4, 4, [4, 3, 2, 1], True, False, True),
(4, 16, [1, 5, 10, 16], False, False, True),
(4, 16, [16, 10, 5, 1], True, True, False),
],
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_multi_token_matches_python(
batch_size: int,
seq_len: int,
accepted_counts: list[int],
has_bias: bool,
silu_activation: bool,
is_vnni: bool,
) -> None:
dim = 96
state_len = seq_len + 2
x = tensor_cache(batch_size * seq_len * dim, torch.bfloat16).view(
batch_size, seq_len, dim
)
weight = tensor_cache(dim * CONV_KERNEL, torch.bfloat16).view(dim, CONV_KERNEL)
bias = tensor_cache(dim, torch.bfloat16) if has_bias else None
conv_state_indices = torch.arange(batch_size - 1, -1, -1, dtype=torch.int32)
num_accepted_tokens = torch.tensor(accepted_counts, dtype=torch.int32)
conv_states_ref = _sd_conv_states(batch_size, state_len, dim)
conv_states_ref.copy_(
tensor_cache(conv_states_ref.numel(), torch.bfloat16).view_as(conv_states_ref)
)
conv_states = conv_states_ref.clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out = ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
weight=conv_weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
num_accepted_tokens=num_accepted_tokens,
)
ref_out = _ref_causal_conv1d_update_cpu_multi(
x=x,
conv_states=conv_states_ref,
weight=weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
)
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(conv_states, conv_states_ref, atol=0, rtol=0)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize("num_accepted", [0, 17])
@torch.inference_mode()
def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
num_accepted: int,
) -> None:
batch_size = 1
seq_len = 16
dim = 96
state_len = seq_len + 2
x = torch.zeros(batch_size, seq_len, dim, dtype=torch.bfloat16)
weight = torch.zeros(dim, CONV_KERNEL, dtype=torch.bfloat16)
conv_states = _sd_conv_states(batch_size, state_len, dim)
with pytest.raises(RuntimeError, match="num_accepted_tokens must be in.*seqlen"):
ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
weight=weight,
bias=None,
silu_activation=True,
conv_state_indices=torch.tensor([0], dtype=torch.int32),
is_vnni=False,
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
@@ -515,6 +893,62 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
is_vnni = True
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
total_tokens = sum(seq_lens)
x, weight, bias = _conv_inputs(total_tokens)
query_start_loc = torch.tensor([0, seq_lens[0], total_tokens], dtype=torch.int32)
cache_indices = torch.tensor([2, 0], dtype=torch.int32)
has_initial_state = torch.tensor([True, False])
narrow_state = _sd_conv_states(3, state_len)
narrow_state.copy_(
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
)
wide_state = _sd_conv_states(3, wide_state_len)
wide_state[:, :, :state_len].copy_(narrow_state)
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out_narrow = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
bias=bias,
conv_states=narrow_state,
query_start_loc=query_start_loc,
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
)
out_wide = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
bias=bias,
conv_states=wide_state,
query_start_loc=query_start_loc,
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
)
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
@torch.inference_mode()
def test_batch_memcpy_cpu_fallback() -> None:
"""The ctypes batch_memcpy fallback (used when triton-cpu is absent) must