[CPU][Spec Decode] Optimize GDN conv path for speculative decoding (#48577)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Tianmu Li
2026-07-27 06:20:14 +00:00
committed by GitHub
co-authored by Codex Li, Jiang <jiang1.li@intel.com>
parent c314af1abf
commit 544cb724c8
5 changed files with 767 additions and 122 deletions
+173 -12
View File
@@ -451,6 +451,90 @@ void causal_conv1d_update_kernel_impl(
});
}
template <typename scalar_t>
void causal_conv1d_update_multi_kernel_impl(
scalar_t* __restrict__ out,
const scalar_t* __restrict__ input,
scalar_t* __restrict__ conv_states,
const scalar_t* __restrict__ weight,
const scalar_t* __restrict__ bias,
const int32_t* __restrict__ num_accepted_tokens,
const int32_t* __restrict__ conv_indices,
bool silu_activation,
int64_t batch,
int64_t dim,
int64_t seqlen,
int64_t width,
int64_t state_len,
int64_t conv_state_slot_stride) {
constexpr int64_t BLOCK_N = block_size_n() * 2;
const int64_t NB = div_up(dim, BLOCK_N);
AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] {
at::parallel_for(0, batch * NB, 0, [&](int64_t begin, int64_t end) {
int64_t bs{0}, nb{0};
data_index_init(begin, bs, batch, nb, NB);
for (int64_t i = begin; i < end; ++i) {
const int64_t nb_start = nb * BLOCK_N;
const int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
const int32_t conv_state_index = conv_indices[bs];
const int32_t history_offset = num_accepted_tokens[bs] - 1;
switch (width << 4 | nb_size >> 4) {
case 0x42:
tinygemm_kernel<scalar_t, 4, 32, has_bias, has_silu>::apply(
input + bs * seqlen * dim + nb_start,
weight + nb_start * width,
out + bs * seqlen * dim + nb_start,
has_bias ? bias + nb_start : nullptr,
conv_states + conv_state_index * conv_state_slot_stride +
history_offset * dim + nb_start,
true,
seqlen,
dim,
true);
break;
case 0x44:
tinygemm_kernel<scalar_t, 4, 64, has_bias, has_silu>::apply(
input + bs * seqlen * dim + nb_start,
weight + nb_start * width,
out + bs * seqlen * dim + nb_start,
has_bias ? bias + nb_start : nullptr,
conv_states + conv_state_index * conv_state_slot_stride +
history_offset * dim + nb_start,
true,
seqlen,
dim,
true);
break;
default:
TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size);
}
data_index_step(bs, batch, nb, NB);
}
});
});
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
for (int64_t bs = begin; bs < end; ++bs) {
const int32_t conv_state_index = conv_indices[bs];
const int32_t num_accepted = num_accepted_tokens[bs];
scalar_t* state = conv_states + conv_state_index * conv_state_slot_stride;
std::memmove(
state,
state + num_accepted * dim,
(state_len - seqlen) * dim * sizeof(scalar_t));
std::memcpy(
state + (state_len - seqlen) * dim,
input + bs * seqlen * dim,
seqlen * dim * sizeof(scalar_t));
}
});
}
} // anonymous namespace
// from [dim, width] or [N, K]
@@ -545,7 +629,7 @@ at::Tensor get_block_indices(const std::optional<at::Tensor>& offsets, int64_t n
// query_start_loc: (batch + 1) int32
// cache_indices: (batch) int32
// has_initial_state: (batch) bool
// conv_states: (..., dim, width - 1) itype
// conv_states: (..., dim, state_len) itype, where state_len >= width - 1
// activation: either None or "silu" or "swish"
// pad_slot_id: int
//
@@ -586,11 +670,14 @@ at::Tensor causal_conv1d_fwd_cpu(
CHECK_EQ(conv_states_val.scalar_type(), scalar_type);
CHECK_GE(padded_batch, batch);
CHECK_EQ(conv_states_val.size(1), dim);
CHECK_EQ(conv_states_val.size(2), width - 1);
const int64_t state_len = conv_states_val.size(2);
CHECK_GE(state_len, width - 1);
// adjust `conv_states` to be contiguous on `dim`
// should happen only once
if (conv_states_val.stride(-2) != 1) {
TORCH_CHECK(state_len == width - 1,
"causal_conv1d_fwd_cpu: wide conv_states must be contiguous on dim.");
auto conv_states_copy = conv_states_val.clone();
conv_states_val.as_strided_({padded_batch, dim, width - 1}, {(width - 1) * dim, 1, dim});
conv_states_val.copy_(conv_states_copy);
@@ -651,14 +738,14 @@ at::Tensor causal_conv1d_fwd_cpu(
// API aligned with GPUs
//
// x: (batch, dim) or (batch, dim, seqlen)
// x: (batch, dim) or (batch, seqlen, dim)
// conv_state: (..., dim, state_len), where state_len >= width - 1
// weight: (dim, width)
// bias: (dim,)
// cache_seqlens: (batch,), dtype int32.
// num_accepted_tokens: (batch,), dtype int32.
// conv_state_indices: (batch,), dtype int32
// pad_slot_id: int
// out: (batch, dim) or (batch, dim, seqlen)
// out: (batch, dim) or (batch, seqlen, dim)
//
at::Tensor causal_conv1d_update_cpu(
const at::Tensor& x,
@@ -666,7 +753,7 @@ at::Tensor causal_conv1d_update_cpu(
const at::Tensor& weight,
const std::optional<at::Tensor>& bias,
bool silu_activation,
const std::optional<at::Tensor>& cache_seqlens,
const std::optional<at::Tensor>& num_accepted_tokens,
const std::optional<at::Tensor>& conv_state_indices,
int64_t pad_slot_id,
bool is_vnni) {
@@ -674,13 +761,13 @@ at::Tensor causal_conv1d_update_cpu(
CHECK_CONTIGUOUS(weight);
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
// TODO: add multi-token prediction support
TORCH_CHECK(x.dim() == 2, "causal_conv1d_update_cpu: expect x to be 2D tensor.");
TORCH_CHECK(!cache_seqlens.has_value(), "causal_conv1d_update_cpu: don't support cache_seqlens.");
TORCH_CHECK(
x.dim() == 2 || x.dim() == 3,
"causal_conv1d_update_cpu: expect x to be 2D or 3D tensor.");
int64_t batch = x.size(0);
int64_t dim = x.size(1);
int64_t seqlen = 1;
int64_t dim = x.dim() == 2 ? x.size(1) : x.size(2);
int64_t seqlen = x.dim() == 2 ? 1 : x.size(1);
int64_t width = weight.size(-1);
const auto scalar_type = x.scalar_type();
@@ -690,10 +777,84 @@ at::Tensor causal_conv1d_update_cpu(
CHECK_EQ(conv_states.scalar_type(), scalar_type);
CHECK_EQ(conv_states.size(1), dim);
CHECK_EQ(conv_states.size(2), width - 1);
const int64_t state_len = conv_states.size(2);
CHECK_GE(state_len, width - 1);
if (x.dim() == 3) {
TORCH_CHECK(
num_accepted_tokens.has_value(),
"causal_conv1d_update_cpu: num_accepted_tokens is required for 3D x.");
TORCH_CHECK(
conv_state_indices.has_value(),
"causal_conv1d_update_cpu: conv_state_indices is required for 3D x.");
CHECK_OPTIONAL_SHAPE_DTYPE(num_accepted_tokens, batch, at::kInt);
TORCH_CHECK(
width == 4,
"causal_conv1d_update_cpu: support only width of 4 for 3D x.");
TORCH_CHECK(
seqlen > 0,
"causal_conv1d_update_cpu: expect non-empty sequence for 3D x.");
TORCH_CHECK(
state_len >= seqlen,
"causal_conv1d_update_cpu: state_len must be >= seqlen for 3D x.");
TORCH_CHECK(
conv_states.stride(-2) == 1 && conv_states.stride(-1) == dim,
"causal_conv1d_update_cpu: 3D x requires SD conv_states layout.");
const int32_t* accepted_counts =
num_accepted_tokens.value().data_ptr<int32_t>();
const int32_t* indices = conv_state_indices.value().data_ptr<int32_t>();
const int64_t num_slots = conv_states.size(0);
for (int64_t bs = 0; bs < batch; ++bs) {
const int32_t num_accepted = accepted_counts[bs];
const int32_t conv_state_index = indices[bs];
TORCH_CHECK(
conv_state_index != pad_slot_id,
"causal_conv1d_update_cpu: 3D x does not support pad slots.");
TORCH_CHECK(
conv_state_index >= 0 && conv_state_index < num_slots,
"causal_conv1d_update_cpu: conv_state_indices out of range.");
TORCH_CHECK(
num_accepted >= 1 && num_accepted <= seqlen,
"causal_conv1d_update_cpu: num_accepted_tokens must be in [1, "
"seqlen].");
TORCH_CHECK(
num_accepted - 1 + width - 1 <= state_len,
"causal_conv1d_update_cpu: history window exceeds conv_states.");
}
int64_t conv_state_slot_stride = conv_states.stride(0);
at::Tensor out = at::empty_like(x);
AT_DISPATCH_REDUCED_FLOATING_TYPES(
scalar_type, "causal_conv1d_update_multi_kernel_impl", [&] {
causal_conv1d_update_multi_kernel_impl<scalar_t>(
out.data_ptr<scalar_t>(),
x.data_ptr<scalar_t>(),
conv_states.data_ptr<scalar_t>(),
packed_w.data_ptr<scalar_t>(),
conditional_data_ptr<scalar_t>(bias),
accepted_counts,
indices,
silu_activation,
batch,
dim,
seqlen,
width,
state_len,
conv_state_slot_stride);
});
return out;
}
TORCH_CHECK(
!num_accepted_tokens.has_value(),
"causal_conv1d_update_cpu: num_accepted_tokens is only supported for 3D "
"x.");
// adjust `conv_states` to be contiguous on `dim`
if (conv_states.stride(-2) != 1) {
TORCH_CHECK(state_len == width - 1,
"causal_conv1d_update_cpu: wide conv_states must be contiguous on dim.");
int64_t num_cache_lines = conv_states.size(0);
auto conv_states_copy = conv_states.clone();
conv_states.as_strided_({num_cache_lines, dim, width - 1}, {(width - 1) * dim, 1, dim});
+3 -2
View File
@@ -147,7 +147,7 @@ at::Tensor causal_conv1d_fwd_cpu(
at::Tensor causal_conv1d_update_cpu(
const at::Tensor& x, const at::Tensor& conv_states,
const at::Tensor& weight, const std::optional<at::Tensor>& bias,
bool silu_activation, const std::optional<at::Tensor>& cache_seqlens,
bool silu_activation, const std::optional<at::Tensor>& num_accepted_tokens,
const std::optional<at::Tensor>& conv_state_indices, int64_t pad_slot_id,
bool is_vnni);
@@ -516,7 +516,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
ops.def(
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
"weight, Tensor? bias, bool silu_activation,"
"Tensor? cache_seqlens, Tensor? conv_state_indices, int pad_slot_id, "
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
"pad_slot_id, "
"bool is_vnni) -> Tensor");
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
#endif
+435 -1
View File
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import types
import pytest
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from vllm.model_executor.layers.mamba.ops.cpu import gdn_attention
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -417,6 +419,207 @@ def _conv_inputs(total_tokens: int):
return x, weight, bias
def _sd_conv_states(
num_slots: int, state_len: int, dim: int = CONV_DIM
) -> torch.Tensor:
storage = torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16)
return storage.transpose(1, 2)
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
@torch.inference_mode()
def test_spec_aware_mixed_routing_preserves_token_order(
monkeypatch: pytest.MonkeyPatch,
) -> None:
num_tokens = 4
projection = torch.arange(num_tokens * 16, dtype=torch.float32).view(num_tokens, 16)
mixed_qkv, b, a = projection[:, :4], projection[:, 4:6], projection[:, 6:8]
assert all(not tensor.is_contiguous() for tensor in (mixed_qkv, b, a))
spec_indices = torch.tensor([0, 2])
nonspec_indices = torch.tensor([1, 3])
metadata = types.SimpleNamespace(
spec_sequence_masks=torch.ones(1, dtype=torch.bool),
spec_token_indx=spec_indices,
non_spec_token_indx=nonspec_indices,
num_prefills=1,
num_decodes=0,
)
routed = []
def record(*args):
routed.append(args[2:5])
return args[2]
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
monkeypatch.setattr(gdn_attention, "_spec_forward", record)
monkeypatch.setattr(gdn_attention, "_spec_aware_nonspec_subset", record)
layer = types.SimpleNamespace(
kv_cache=[torch.empty(1, 4, 6), torch.empty(1, 1, 1, 1)]
)
core_attn_out = torch.empty_like(mixed_qkv)
gdn_attention._cpu_gdn_attention_spec_aware(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv=mixed_qkv,
b=b,
a=a,
core_attn_out=core_attn_out,
width=CONV_KERNEL,
state_len=6,
)
expected_inputs = (mixed_qkv, b, a)
assert len(routed) == 2
for actual_inputs, indices in zip(routed, (spec_indices, nonspec_indices)):
for actual, expected in zip(actual_inputs, expected_inputs):
assert actual.is_contiguous()
torch.testing.assert_close(actual, expected.index_select(0, indices))
torch.testing.assert_close(core_attn_out, mixed_qkv)
@torch.inference_mode()
def test_spec_aware_nonspec_materializes_state_indices(
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.arange(8, dtype=torch.int32).view(2, 4)
state_indices = block_table[:, 0]
assert not state_indices.is_contiguous()
metadata = types.SimpleNamespace(
non_spec_state_indices_tensor=state_indices,
non_spec_query_start_loc=torch.tensor([0, 2, 4], dtype=torch.int32),
num_decodes=0,
num_decode_tokens=0,
num_prefills=2,
num_prefill_tokens=4,
has_initial_state=torch.tensor([False, False]),
)
recorded_indices = None
def causal_conv1d_fwd_cpu(**kwargs):
nonlocal recorded_indices
recorded_indices = kwargs["cache_indices"]
return kwargs["x"]
def fused_gdn_gating_cpu(**kwargs):
return kwargs["a"], kwargs["b"]
def chunk_gated_delta_rule_cpu(**kwargs):
out = torch.zeros(1, 4, 1, 1)
return out, kwargs["initial_state"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_fwd_cpu", causal_conv1d_fwd_cpu
)
monkeypatch.setattr(gdn_attention.ops, "fused_gdn_gating_cpu", fused_gdn_gating_cpu)
monkeypatch.setattr(
gdn_attention.ops,
"chunk_gated_delta_rule_cpu",
chunk_gated_delta_rule_cpu,
)
layer = types.SimpleNamespace(
activation="silu",
conv1d=types.SimpleNamespace(weight=torch.empty(0), bias=None),
A_log=torch.empty(0),
dt_bias=torch.empty(0),
rearrange_mixed_qkv=lambda x: (
x[:, :1].view(1, 4, 1, 1),
x[:, :1].view(1, 4, 1, 1),
x[:, :1].view(1, 4, 1, 1),
),
)
gdn_attention._spec_aware_nonspec(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv=torch.zeros(4, 4),
b=torch.zeros(4, 1),
a=torch.zeros(4, 1),
core_attn_out=torch.zeros(4, 1, 1),
conv_buf=torch.zeros(8, 1, 6),
ssm_state=torch.zeros(8, 1, 1, 1),
width=4,
)
assert recorded_indices is not None
assert recorded_indices.is_contiguous()
torch.testing.assert_close(
recorded_indices, torch.tensor([0, 4], dtype=torch.int32)
)
@torch.inference_mode()
def test_spec_forward_prepares_native_conv_metadata(
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
state_indices = block_table[:, 0]
accepted_counts = torch.tensor([1, 4], dtype=torch.int32)
assert not state_indices.is_contiguous()
metadata = types.SimpleNamespace(
num_spec_decodes=2,
spec_state_indices_tensor=block_table,
spec_query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32),
num_accepted_tokens=accepted_counts,
)
forwarded_indices = None
forwarded_counts = None
def causal_conv1d_update_cpu(**kwargs):
nonlocal forwarded_counts, forwarded_indices
forwarded_indices = kwargs["conv_state_indices"]
forwarded_counts = kwargs["num_accepted_tokens"]
return kwargs["x"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
)
monkeypatch.setattr(
gdn_attention.ops,
"fused_sigmoid_gating_delta_rule_update_spec_cpu",
lambda **kwargs: kwargs["q"],
)
layer = types.SimpleNamespace(
activation="silu",
conv1d=types.SimpleNamespace(weight=torch.empty(1, CONV_KERNEL), bias=None),
A_log=None,
dt_bias=None,
rearrange_mixed_qkv=lambda x: (x.unsqueeze(0),) * 3,
)
gdn_attention._spec_forward(
layer=layer,
attn_metadata_i=metadata,
mixed_qkv_spec=torch.zeros(8, 1, dtype=torch.bfloat16),
b_spec=torch.empty(0),
a_spec=torch.empty(0),
conv_buf=torch.empty(0),
ssm_state=torch.empty(0),
width=CONV_KERNEL,
state_len=0,
)
expected = (
(forwarded_indices, torch.tensor([0, 4], dtype=torch.int32)),
(forwarded_counts, torch.tensor([1, 4], dtype=torch.int32)),
)
for actual, reference in expected:
assert actual is not None
assert actual.is_contiguous()
assert actual.dtype == torch.int32
torch.testing.assert_close(actual, reference)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> None:
@@ -475,7 +678,182 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="causal_conv1d_fwd_cpu requires AMX/AVX512",
reason="requires AMX support",
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
batch_size = 3
is_vnni = True
x, weight, bias = _conv_inputs(batch_size)
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
narrow_state = _sd_conv_states(batch_size, state_len)
narrow_state.copy_(
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
)
wide_state = _sd_conv_states(batch_size, wide_state_len)
wide_state[:, :, :state_len].copy_(narrow_state)
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out_narrow = ops.causal_conv1d_update_cpu(
x=x,
conv_states=narrow_state,
weight=conv_weight,
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
)
out_wide = ops.causal_conv1d_update_cpu(
x=x,
conv_states=wide_state,
weight=conv_weight,
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
)
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
def _ref_causal_conv1d_update_cpu_multi(
x: torch.Tensor,
conv_states: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
) -> torch.Tensor:
batch_size, seq_len, dim = x.shape
state_len = conv_states.size(2)
conv_out = torch.empty_like(x)
conv_weight = weight.unsqueeze(1)
for i in range(batch_size):
slot = int(conv_state_indices[i].item())
offset = int(num_accepted_tokens[i].item()) - 1
state = conv_states[slot]
x_seq = x[i].transpose(0, 1).to(state.dtype)
prior = state[:, offset : offset + CONV_KERNEL - 1]
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0)
out = F.conv1d(conv_in, conv_weight, bias, groups=dim)[0]
if silu_activation:
out = F.silu(out)
conv_out[i] = out.transpose(0, 1).to(conv_out.dtype)
keep = state[:, offset + 1 : offset + 1 + (state_len - seq_len)]
state.copy_(torch.cat([keep, x_seq], dim=-1))
return conv_out
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize(
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
[
(1, 1, [1], False, False, False),
(1, 1, [1], True, True, True),
(4, 4, [1, 2, 3, 4], False, True, False),
(4, 4, [4, 3, 2, 1], True, False, True),
(4, 16, [1, 5, 10, 16], False, False, True),
(4, 16, [16, 10, 5, 1], True, True, False),
],
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_multi_token_matches_python(
batch_size: int,
seq_len: int,
accepted_counts: list[int],
has_bias: bool,
silu_activation: bool,
is_vnni: bool,
) -> None:
dim = 96
state_len = seq_len + 2
x = tensor_cache(batch_size * seq_len * dim, torch.bfloat16).view(
batch_size, seq_len, dim
)
weight = tensor_cache(dim * CONV_KERNEL, torch.bfloat16).view(dim, CONV_KERNEL)
bias = tensor_cache(dim, torch.bfloat16) if has_bias else None
conv_state_indices = torch.arange(batch_size - 1, -1, -1, dtype=torch.int32)
num_accepted_tokens = torch.tensor(accepted_counts, dtype=torch.int32)
conv_states_ref = _sd_conv_states(batch_size, state_len, dim)
conv_states_ref.copy_(
tensor_cache(conv_states_ref.numel(), torch.bfloat16).view_as(conv_states_ref)
)
conv_states = conv_states_ref.clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out = ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
weight=conv_weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
num_accepted_tokens=num_accepted_tokens,
)
ref_out = _ref_causal_conv1d_update_cpu_multi(
x=x,
conv_states=conv_states_ref,
weight=weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
)
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(conv_states, conv_states_ref, atol=0, rtol=0)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize("num_accepted", [0, 17])
@torch.inference_mode()
def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
num_accepted: int,
) -> None:
batch_size = 1
seq_len = 16
dim = 96
state_len = seq_len + 2
x = torch.zeros(batch_size, seq_len, dim, dtype=torch.bfloat16)
weight = torch.zeros(dim, CONV_KERNEL, dtype=torch.bfloat16)
conv_states = _sd_conv_states(batch_size, state_len, dim)
with pytest.raises(RuntimeError, match="num_accepted_tokens must be in.*seqlen"):
ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
weight=weight,
bias=None,
silu_activation=True,
conv_state_indices=torch.tensor([0], dtype=torch.int32),
is_vnni=False,
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
@@ -515,6 +893,62 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
is_vnni = True
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
total_tokens = sum(seq_lens)
x, weight, bias = _conv_inputs(total_tokens)
query_start_loc = torch.tensor([0, seq_lens[0], total_tokens], dtype=torch.int32)
cache_indices = torch.tensor([2, 0], dtype=torch.int32)
has_initial_state = torch.tensor([True, False])
narrow_state = _sd_conv_states(3, state_len)
narrow_state.copy_(
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
)
wide_state = _sd_conv_states(3, wide_state_len)
wide_state[:, :, :state_len].copy_(narrow_state)
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
out_narrow = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
bias=bias,
conv_states=narrow_state,
query_start_loc=query_start_loc,
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
)
out_wide = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
bias=bias,
conv_states=wide_state,
query_start_loc=query_start_loc,
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
)
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
@torch.inference_mode()
def test_batch_memcpy_cpu_fallback() -> None:
"""The ctypes batch_memcpy fallback (used when triton-cpu is absent) must
+2 -1
View File
@@ -3505,6 +3505,7 @@ def causal_conv1d_update_cpu(
silu_activation: bool,
conv_state_indices: torch.Tensor | None,
is_vnni: bool,
num_accepted_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.ops._C.causal_conv1d_update_cpu(
x,
@@ -3512,7 +3513,7 @@ def causal_conv1d_update_cpu(
weight,
bias,
silu_activation,
None,
num_accepted_tokens,
conv_state_indices,
-1,
is_vnni,
@@ -306,25 +306,21 @@ def _cpu_gdn_attention_spec_aware(
width: int,
state_len: int,
) -> None:
mixed_qkv = mixed_qkv.contiguous()
a = a.contiguous()
b = b.contiguous()
spec_sequence_masks = attn_metadata_i.spec_sequence_masks
conv_buf = _conv_buffer_view(layer) # (num_slots, dim, state_len)
ssm_state = _ssm_state_view(layer)
if spec_sequence_masks is None:
# No spec sequences in this batch (e.g. the prompt prefill step while
# speculative decoding is configured). Process as prefill/decode using
# torch conv (which only touches the first ``width-1`` columns of the
# wide buffer, leaving the rolling history untouched).
# speculative decoding is configured). Process as ordinary
# prefill/decode while touching only the first ``width-1`` columns of
# the wide buffer, leaving the rolling history untouched.
_spec_aware_nonspec(
layer,
attn_metadata_i,
mixed_qkv,
b,
a,
mixed_qkv.contiguous(),
b.contiguous(),
a.contiguous(),
core_attn_out,
conv_buf,
ssm_state,
@@ -339,9 +335,9 @@ def _cpu_gdn_attention_spec_aware(
num_decodes = attn_metadata_i.num_decodes
if num_prefills == 0 and num_decodes == 0:
mixed_qkv_spec = mixed_qkv
b_spec = b
a_spec = a
mixed_qkv_spec = mixed_qkv.contiguous()
b_spec = b.contiguous()
a_spec = a.contiguous()
spec_out_indx = None
else:
assert spec_token_indx is not None
@@ -405,50 +401,71 @@ def _spec_forward(
assert spec_qsl is not None
assert num_accepted is not None
spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1].to("cpu", torch.int64)
num_acc_cpu = num_accepted[:num_spec_decodes].to("cpu", torch.int64)
spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1]
seq_starts = spec_qsl_cpu[:-1]
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
# ---- 1. Convolution (per-sequence rolling buffer) ----
w2d = _unpacked_conv_weight(layer) # (dim, width)
dim = w2d.size(0)
w = w2d.unsqueeze(1) # (dim, 1, width) for F.conv1d depthwise
dim = mixed_qkv_spec.size(-1)
bias = layer.conv1d.bias
silu = layer.activation == "silu"
conv_out = torch.empty_like(mixed_qkv_spec)
col0 = spec_state_indices[:, 0].to("cpu", torch.int64)
for i in range(num_spec_decodes):
q_i = int(seq_lens[i].item())
if q_i == 0:
continue
start = int(seq_starts[i].item())
slot0 = int(col0[i].item())
a_prev = int(num_acc_cpu[i].item())
offset = a_prev - 1
B = conv_buf[slot0] # (dim, state_len)
x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype)
prior = B[:, offset : offset + (width - 1)]
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) # (1, dim, w-1+q)
out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i)
if silu:
out = F.silu(out)
conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype)
# Roll the buffer: drop ``a_prev`` from the front, append the new
# draft tokens, keep total length == state_len.
keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)]
new_B = torch.cat([keep, x_seq], dim=-1)
B.copy_(new_B)
can_use_native_conv = (
torch.cpu._is_amx_tile_supported()
and not is_conv_state_dim_first()
and width == 4
and num_spec_decodes > 0
and bool(torch.all(seq_lens == seq_lens[0]).item())
and int(seq_lens[0].item()) > 0
)
if can_use_native_conv:
q_i = int(seq_lens[0].item())
conv_out = ops.causal_conv1d_update_cpu(
x=mixed_qkv_spec.view(num_spec_decodes, q_i, dim),
conv_states=conv_buf,
weight=layer.conv1d.weight,
bias=bias,
silu_activation=silu,
conv_state_indices=spec_state_indices[:num_spec_decodes, 0]
.to("cpu", torch.int32)
.contiguous(),
is_vnni=True,
num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32),
).view_as(mixed_qkv_spec)
else:
w = _unpacked_conv_weight(layer).unsqueeze(1)
col0 = spec_state_indices[:num_spec_decodes, 0]
num_acc_cpu = num_accepted[:num_spec_decodes]
conv_out = torch.empty_like(mixed_qkv_spec)
for i in range(num_spec_decodes):
q_i = int(seq_lens[i].item())
if q_i == 0:
continue
start = int(seq_starts[i].item())
slot0 = int(col0[i].item())
offset = int(num_acc_cpu[i].item()) - 1
B = conv_buf[slot0] # (dim, state_len)
x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype)
prior = B[:, offset : offset + (width - 1)]
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0)
out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i)
if silu:
out = F.silu(out)
conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype)
# Roll the buffer: drop the accepted history from the front, append
# the new draft tokens, keep total length == state_len.
keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)]
new_B = torch.cat([keep, x_seq], dim=-1)
B.copy_(new_B)
# ---- 2. Recurrent (multi-slot SSM state) ----
# Single fused kernel call: it runs the recurrence over each sequence's
# draft tokens internally, resumes from slot ``num_accepted-1`` and stores
# the state after token ``t`` into slot ``t`` (rollback for the next step).
query, key, value = layer.rearrange_mixed_qkv(conv_out)
query = query.squeeze(0).contiguous()
key = key.squeeze(0).contiguous()
value = value.squeeze(0).contiguous()
query = query.squeeze(0)
key = key.squeeze(0)
value = value.squeeze(0)
spec_idx = spec_state_indices[:num_spec_decodes].to(torch.int32).contiguous()
num_acc = num_accepted[:num_spec_decodes].to(torch.int32).contiguous()
cu = spec_qsl[: num_spec_decodes + 1].to(torch.int32).contiguous()
@@ -458,8 +475,8 @@ def _spec_forward(
q=query,
k=key,
v=value,
a=a_spec.contiguous(),
b=b_spec.contiguous(),
a=a_spec,
b=b_spec,
initial_state_source=ssm_state,
spec_state_indices=spec_idx,
num_accepted_tokens=num_acc,
@@ -469,15 +486,6 @@ def _spec_forward(
return out_spec
def core_attn_out_like(layer, mixed_qkv_spec: torch.Tensor) -> torch.Tensor:
num_tokens = mixed_qkv_spec.size(0)
return torch.zeros(
(num_tokens, layer.num_v_heads // layer.tp_size, layer.head_v_dim),
dtype=mixed_qkv_spec.dtype,
device=mixed_qkv_spec.device,
)
def _spec_aware_nonspec(
layer,
attn_metadata_i: GDNAttentionMetadata,
@@ -489,13 +497,19 @@ def _spec_aware_nonspec(
ssm_state: torch.Tensor,
width: int,
) -> None:
"""Non-spec prefill/decode with a wide conv buffer (torch path)."""
"""Non-spec prefill/decode with a wide conv buffer."""
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
query_start_loc = attn_metadata_i.non_spec_query_start_loc
assert state_indices_tensor is not None
assert query_start_loc is not None
state_indices_tensor = state_indices_tensor.contiguous()
conv_weights = _unpacked_conv_weight(layer)
is_amx = torch.cpu._is_amx_tile_supported()
if is_amx and is_conv_state_dim_first():
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
if not is_amx:
conv_weights = _unpacked_conv_weight(layer)
num_decodes = attn_metadata_i.num_decodes
num_decode_tokens = attn_metadata_i.num_decode_tokens
@@ -507,42 +521,48 @@ def _spec_aware_nonspec(
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
# Only the first ``width-1`` columns hold the real conv state.
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
conv_state_view = conv_buf[:, :, : width - 1]
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
if is_amx:
decode_mixed_qkv = ops.causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_buf[:, :, : width - 1],
weight=conv_weights,
conv_states=conv_buf,
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
activation=layer.activation,
silu_activation=layer.activation == "silu",
conv_state_indices=decode_state_indices,
is_vnni=True,
)
else:
# Only the first ``width-1`` columns hold the real conv state.
conv_state_view = conv_buf[:, :, : width - 1]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
# rearrange_mixed_qkv can return views whose last dim is not
# contiguous; the fused CPU kernel requires a contiguous last dim.
query = query.contiguous()
key = key.contiguous()
value = value.contiguous()
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
A_log=layer.A_log,
dt_bias=layer.dt_bias,
q=query,
k=key,
v=value,
a=decode_a.contiguous(),
b=decode_b.contiguous(),
a=decode_a,
b=decode_b,
initial_state_source=ssm_state,
initial_state_indices=decode_state_indices,
cu_seqlens=query_start_loc[: num_decodes + 1],
@@ -568,17 +588,29 @@ def _spec_aware_nonspec(
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
# ``causal_conv1d_torch`` only touches columns [:width-1] of the buffer.
prefill_mixed_qkv = causal_conv1d_torch(
x=prefill_mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_query_start_loc,
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
activation=layer.activation,
).transpose(0, 1)
if is_amx:
prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_query_start_loc,
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
x=prefill_mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_query_start_loc,
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
activation=layer.activation,
).transpose(0, 1)
query, key, value = layer.rearrange_mixed_qkv(prefill_mixed_qkv)
g, beta = ops.fused_gdn_gating_cpu(
@@ -618,24 +650,41 @@ def _spec_aware_nonspec_subset(
Returns outputs ordered like ``non_spec_token_indx``.
"""
out = core_attn_out_like(layer, mixed_qkv)
has_initial_state = attn_metadata_i.has_initial_state
prefill_state_indices = attn_metadata_i.prefill_state_indices
prefill_qsl = attn_metadata_i.prefill_query_start_loc
assert prefill_state_indices is not None and prefill_qsl is not None
assert has_initial_state is not None
prefill_state_indices = prefill_state_indices.contiguous()
conv_weights = _unpacked_conv_weight(layer)
conv_out = causal_conv1d_torch(
x=mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_qsl,
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
activation=layer.activation,
).transpose(0, 1)
is_amx = torch.cpu._is_amx_tile_supported()
if is_amx and is_conv_state_dim_first():
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
if is_amx:
conv_out = ops.causal_conv1d_fwd_cpu(
x=mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_qsl,
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
).transpose(0, 1)
else:
conv_weights = _unpacked_conv_weight(layer)
conv_out = causal_conv1d_torch(
x=mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_qsl,
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
activation=layer.activation,
).transpose(0, 1)
query, key, value = layer.rearrange_mixed_qkv(conv_out)
g, beta = ops.fused_gdn_gating_cpu(
@@ -658,8 +707,7 @@ def _spec_aware_nonspec_subset(
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
)
out[:] = attn_out.squeeze(0)
return out
return attn_out.squeeze(0)
def cpu_gdn_attention_core_fake(