From 544cb724c888dda1eff06a02cb0baca783c3dd86 Mon Sep 17 00:00:00 2001 From: Tianmu Li Date: Sun, 26 Jul 2026 23:20:14 -0700 Subject: [PATCH] [CPU][Spec Decode] Optimize GDN conv path for speculative decoding (#48577) Signed-off-by: Li, Tianmu Co-authored-by: Codex Co-authored-by: Li, Jiang --- csrc/cpu/sgl-kernels/conv.cpp | 185 +++++++- csrc/cpu/torch_bindings.cpp | 5 +- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py | 436 +++++++++++++++++- vllm/_custom_ops.py | 3 +- .../layers/mamba/ops/cpu/gdn_attention.py | 260 ++++++----- 5 files changed, 767 insertions(+), 122 deletions(-) diff --git a/csrc/cpu/sgl-kernels/conv.cpp b/csrc/cpu/sgl-kernels/conv.cpp index b918aed8bf..8cac9ff1f0 100644 --- a/csrc/cpu/sgl-kernels/conv.cpp +++ b/csrc/cpu/sgl-kernels/conv.cpp @@ -451,6 +451,90 @@ void causal_conv1d_update_kernel_impl( }); } +template +void causal_conv1d_update_multi_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + scalar_t* __restrict__ conv_states, + const scalar_t* __restrict__ weight, + const scalar_t* __restrict__ bias, + const int32_t* __restrict__ num_accepted_tokens, + const int32_t* __restrict__ conv_indices, + bool silu_activation, + int64_t batch, + int64_t dim, + int64_t seqlen, + int64_t width, + int64_t state_len, + int64_t conv_state_slot_stride) { + constexpr int64_t BLOCK_N = block_size_n() * 2; + const int64_t NB = div_up(dim, BLOCK_N); + + AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] { + at::parallel_for(0, batch * NB, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, nb{0}; + data_index_init(begin, bs, batch, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + const int64_t nb_start = nb * BLOCK_N; + const int64_t nb_size = std::min(dim - nb_start, BLOCK_N); + const int32_t conv_state_index = conv_indices[bs]; + const int32_t history_offset = num_accepted_tokens[bs] - 1; + + switch (width << 4 | nb_size >> 4) { + case 0x42: + tinygemm_kernel::apply( + input + bs * seqlen * dim + nb_start, + weight + nb_start * width, + out + bs * seqlen * dim + nb_start, + has_bias ? bias + nb_start : nullptr, + conv_states + conv_state_index * conv_state_slot_stride + + history_offset * dim + nb_start, + true, + seqlen, + dim, + true); + break; + case 0x44: + tinygemm_kernel::apply( + input + bs * seqlen * dim + nb_start, + weight + nb_start * width, + out + bs * seqlen * dim + nb_start, + has_bias ? bias + nb_start : nullptr, + conv_states + conv_state_index * conv_state_slot_stride + + history_offset * dim + nb_start, + true, + seqlen, + dim, + true); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size); + } + + data_index_step(bs, batch, nb, NB); + } + }); + }); + + at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) { + for (int64_t bs = begin; bs < end; ++bs) { + const int32_t conv_state_index = conv_indices[bs]; + const int32_t num_accepted = num_accepted_tokens[bs]; + scalar_t* state = conv_states + conv_state_index * conv_state_slot_stride; + + std::memmove( + state, + state + num_accepted * dim, + (state_len - seqlen) * dim * sizeof(scalar_t)); + std::memcpy( + state + (state_len - seqlen) * dim, + input + bs * seqlen * dim, + seqlen * dim * sizeof(scalar_t)); + } + }); +} + } // anonymous namespace // from [dim, width] or [N, K] @@ -545,7 +629,7 @@ at::Tensor get_block_indices(const std::optional& offsets, int64_t n // query_start_loc: (batch + 1) int32 // cache_indices: (batch) int32 // has_initial_state: (batch) bool -// conv_states: (..., dim, width - 1) itype +// conv_states: (..., dim, state_len) itype, where state_len >= width - 1 // activation: either None or "silu" or "swish" // pad_slot_id: int // @@ -586,11 +670,14 @@ at::Tensor causal_conv1d_fwd_cpu( CHECK_EQ(conv_states_val.scalar_type(), scalar_type); CHECK_GE(padded_batch, batch); CHECK_EQ(conv_states_val.size(1), dim); - CHECK_EQ(conv_states_val.size(2), width - 1); + const int64_t state_len = conv_states_val.size(2); + CHECK_GE(state_len, width - 1); // adjust `conv_states` to be contiguous on `dim` // should happen only once if (conv_states_val.stride(-2) != 1) { + TORCH_CHECK(state_len == width - 1, + "causal_conv1d_fwd_cpu: wide conv_states must be contiguous on dim."); auto conv_states_copy = conv_states_val.clone(); conv_states_val.as_strided_({padded_batch, dim, width - 1}, {(width - 1) * dim, 1, dim}); conv_states_val.copy_(conv_states_copy); @@ -651,14 +738,14 @@ at::Tensor causal_conv1d_fwd_cpu( // API aligned with GPUs // -// x: (batch, dim) or (batch, dim, seqlen) +// x: (batch, dim) or (batch, seqlen, dim) // conv_state: (..., dim, state_len), where state_len >= width - 1 // weight: (dim, width) // bias: (dim,) -// cache_seqlens: (batch,), dtype int32. +// num_accepted_tokens: (batch,), dtype int32. // conv_state_indices: (batch,), dtype int32 // pad_slot_id: int -// out: (batch, dim) or (batch, dim, seqlen) +// out: (batch, dim) or (batch, seqlen, dim) // at::Tensor causal_conv1d_update_cpu( const at::Tensor& x, @@ -666,7 +753,7 @@ at::Tensor causal_conv1d_update_cpu( const at::Tensor& weight, const std::optional& bias, bool silu_activation, - const std::optional& cache_seqlens, + const std::optional& num_accepted_tokens, const std::optional& conv_state_indices, int64_t pad_slot_id, bool is_vnni) { @@ -674,13 +761,13 @@ at::Tensor causal_conv1d_update_cpu( CHECK_CONTIGUOUS(weight); auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight); - // TODO: add multi-token prediction support - TORCH_CHECK(x.dim() == 2, "causal_conv1d_update_cpu: expect x to be 2D tensor."); - TORCH_CHECK(!cache_seqlens.has_value(), "causal_conv1d_update_cpu: don't support cache_seqlens."); + TORCH_CHECK( + x.dim() == 2 || x.dim() == 3, + "causal_conv1d_update_cpu: expect x to be 2D or 3D tensor."); int64_t batch = x.size(0); - int64_t dim = x.size(1); - int64_t seqlen = 1; + int64_t dim = x.dim() == 2 ? x.size(1) : x.size(2); + int64_t seqlen = x.dim() == 2 ? 1 : x.size(1); int64_t width = weight.size(-1); const auto scalar_type = x.scalar_type(); @@ -690,10 +777,84 @@ at::Tensor causal_conv1d_update_cpu( CHECK_EQ(conv_states.scalar_type(), scalar_type); CHECK_EQ(conv_states.size(1), dim); - CHECK_EQ(conv_states.size(2), width - 1); + const int64_t state_len = conv_states.size(2); + CHECK_GE(state_len, width - 1); + + if (x.dim() == 3) { + TORCH_CHECK( + num_accepted_tokens.has_value(), + "causal_conv1d_update_cpu: num_accepted_tokens is required for 3D x."); + TORCH_CHECK( + conv_state_indices.has_value(), + "causal_conv1d_update_cpu: conv_state_indices is required for 3D x."); + CHECK_OPTIONAL_SHAPE_DTYPE(num_accepted_tokens, batch, at::kInt); + TORCH_CHECK( + width == 4, + "causal_conv1d_update_cpu: support only width of 4 for 3D x."); + TORCH_CHECK( + seqlen > 0, + "causal_conv1d_update_cpu: expect non-empty sequence for 3D x."); + TORCH_CHECK( + state_len >= seqlen, + "causal_conv1d_update_cpu: state_len must be >= seqlen for 3D x."); + TORCH_CHECK( + conv_states.stride(-2) == 1 && conv_states.stride(-1) == dim, + "causal_conv1d_update_cpu: 3D x requires SD conv_states layout."); + + const int32_t* accepted_counts = + num_accepted_tokens.value().data_ptr(); + const int32_t* indices = conv_state_indices.value().data_ptr(); + const int64_t num_slots = conv_states.size(0); + for (int64_t bs = 0; bs < batch; ++bs) { + const int32_t num_accepted = accepted_counts[bs]; + const int32_t conv_state_index = indices[bs]; + TORCH_CHECK( + conv_state_index != pad_slot_id, + "causal_conv1d_update_cpu: 3D x does not support pad slots."); + TORCH_CHECK( + conv_state_index >= 0 && conv_state_index < num_slots, + "causal_conv1d_update_cpu: conv_state_indices out of range."); + TORCH_CHECK( + num_accepted >= 1 && num_accepted <= seqlen, + "causal_conv1d_update_cpu: num_accepted_tokens must be in [1, " + "seqlen]."); + TORCH_CHECK( + num_accepted - 1 + width - 1 <= state_len, + "causal_conv1d_update_cpu: history window exceeds conv_states."); + } + + int64_t conv_state_slot_stride = conv_states.stride(0); + at::Tensor out = at::empty_like(x); + AT_DISPATCH_REDUCED_FLOATING_TYPES( + scalar_type, "causal_conv1d_update_multi_kernel_impl", [&] { + causal_conv1d_update_multi_kernel_impl( + out.data_ptr(), + x.data_ptr(), + conv_states.data_ptr(), + packed_w.data_ptr(), + conditional_data_ptr(bias), + accepted_counts, + indices, + silu_activation, + batch, + dim, + seqlen, + width, + state_len, + conv_state_slot_stride); + }); + return out; + } + + TORCH_CHECK( + !num_accepted_tokens.has_value(), + "causal_conv1d_update_cpu: num_accepted_tokens is only supported for 3D " + "x."); // adjust `conv_states` to be contiguous on `dim` if (conv_states.stride(-2) != 1) { + TORCH_CHECK(state_len == width - 1, + "causal_conv1d_update_cpu: wide conv_states must be contiguous on dim."); int64_t num_cache_lines = conv_states.size(0); auto conv_states_copy = conv_states.clone(); conv_states.as_strided_({num_cache_lines, dim, width - 1}, {(width - 1) * dim, 1, dim}); diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index d5876202f1..80b9409d99 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -147,7 +147,7 @@ at::Tensor causal_conv1d_fwd_cpu( at::Tensor causal_conv1d_update_cpu( const at::Tensor& x, const at::Tensor& conv_states, const at::Tensor& weight, const std::optional& bias, - bool silu_activation, const std::optional& cache_seqlens, + bool silu_activation, const std::optional& num_accepted_tokens, const std::optional& conv_state_indices, int64_t pad_slot_id, bool is_vnni); @@ -516,7 +516,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def( "causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor " "weight, Tensor? bias, bool silu_activation," - "Tensor? cache_seqlens, Tensor? conv_state_indices, int pad_slot_id, " + "Tensor? num_accepted_tokens, Tensor? conv_state_indices, int " + "pad_slot_id, " "bool is_vnni) -> Tensor"); ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu); #endif diff --git a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py index 083e9ff22a..7924b5a1ac 100644 --- a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py +++ b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +import types import pytest import torch import torch.nn.functional as F import vllm._custom_ops as ops +from vllm.model_executor.layers.mamba.ops.cpu import gdn_attention from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed @@ -417,6 +419,207 @@ def _conv_inputs(total_tokens: int): return x, weight, bias +def _sd_conv_states( + num_slots: int, state_len: int, dim: int = CONV_DIM +) -> torch.Tensor: + storage = torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16) + return storage.transpose(1, 2) + + +def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor: + return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight + + +@torch.inference_mode() +def test_spec_aware_mixed_routing_preserves_token_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + num_tokens = 4 + projection = torch.arange(num_tokens * 16, dtype=torch.float32).view(num_tokens, 16) + mixed_qkv, b, a = projection[:, :4], projection[:, 4:6], projection[:, 6:8] + assert all(not tensor.is_contiguous() for tensor in (mixed_qkv, b, a)) + + spec_indices = torch.tensor([0, 2]) + nonspec_indices = torch.tensor([1, 3]) + metadata = types.SimpleNamespace( + spec_sequence_masks=torch.ones(1, dtype=torch.bool), + spec_token_indx=spec_indices, + non_spec_token_indx=nonspec_indices, + num_prefills=1, + num_decodes=0, + ) + routed = [] + + def record(*args): + routed.append(args[2:5]) + return args[2] + + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True) + monkeypatch.setattr(gdn_attention, "_spec_forward", record) + monkeypatch.setattr(gdn_attention, "_spec_aware_nonspec_subset", record) + + layer = types.SimpleNamespace( + kv_cache=[torch.empty(1, 4, 6), torch.empty(1, 1, 1, 1)] + ) + core_attn_out = torch.empty_like(mixed_qkv) + gdn_attention._cpu_gdn_attention_spec_aware( + layer=layer, + attn_metadata_i=metadata, + mixed_qkv=mixed_qkv, + b=b, + a=a, + core_attn_out=core_attn_out, + width=CONV_KERNEL, + state_len=6, + ) + + expected_inputs = (mixed_qkv, b, a) + assert len(routed) == 2 + for actual_inputs, indices in zip(routed, (spec_indices, nonspec_indices)): + for actual, expected in zip(actual_inputs, expected_inputs): + assert actual.is_contiguous() + torch.testing.assert_close(actual, expected.index_select(0, indices)) + torch.testing.assert_close(core_attn_out, mixed_qkv) + + +@torch.inference_mode() +def test_spec_aware_nonspec_materializes_state_indices( + monkeypatch: pytest.MonkeyPatch, +) -> None: + block_table = torch.arange(8, dtype=torch.int32).view(2, 4) + state_indices = block_table[:, 0] + assert not state_indices.is_contiguous() + + metadata = types.SimpleNamespace( + non_spec_state_indices_tensor=state_indices, + non_spec_query_start_loc=torch.tensor([0, 2, 4], dtype=torch.int32), + num_decodes=0, + num_decode_tokens=0, + num_prefills=2, + num_prefill_tokens=4, + has_initial_state=torch.tensor([False, False]), + ) + + recorded_indices = None + + def causal_conv1d_fwd_cpu(**kwargs): + nonlocal recorded_indices + recorded_indices = kwargs["cache_indices"] + return kwargs["x"] + + def fused_gdn_gating_cpu(**kwargs): + return kwargs["a"], kwargs["b"] + + def chunk_gated_delta_rule_cpu(**kwargs): + out = torch.zeros(1, 4, 1, 1) + return out, kwargs["initial_state"] + + monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True) + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False) + monkeypatch.setattr( + gdn_attention.ops, "causal_conv1d_fwd_cpu", causal_conv1d_fwd_cpu + ) + monkeypatch.setattr(gdn_attention.ops, "fused_gdn_gating_cpu", fused_gdn_gating_cpu) + monkeypatch.setattr( + gdn_attention.ops, + "chunk_gated_delta_rule_cpu", + chunk_gated_delta_rule_cpu, + ) + + layer = types.SimpleNamespace( + activation="silu", + conv1d=types.SimpleNamespace(weight=torch.empty(0), bias=None), + A_log=torch.empty(0), + dt_bias=torch.empty(0), + rearrange_mixed_qkv=lambda x: ( + x[:, :1].view(1, 4, 1, 1), + x[:, :1].view(1, 4, 1, 1), + x[:, :1].view(1, 4, 1, 1), + ), + ) + gdn_attention._spec_aware_nonspec( + layer=layer, + attn_metadata_i=metadata, + mixed_qkv=torch.zeros(4, 4), + b=torch.zeros(4, 1), + a=torch.zeros(4, 1), + core_attn_out=torch.zeros(4, 1, 1), + conv_buf=torch.zeros(8, 1, 6), + ssm_state=torch.zeros(8, 1, 1, 1), + width=4, + ) + + assert recorded_indices is not None + assert recorded_indices.is_contiguous() + torch.testing.assert_close( + recorded_indices, torch.tensor([0, 4], dtype=torch.int32) + ) + + +@torch.inference_mode() +def test_spec_forward_prepares_native_conv_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32) + state_indices = block_table[:, 0] + accepted_counts = torch.tensor([1, 4], dtype=torch.int32) + assert not state_indices.is_contiguous() + metadata = types.SimpleNamespace( + num_spec_decodes=2, + spec_state_indices_tensor=block_table, + spec_query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32), + num_accepted_tokens=accepted_counts, + ) + forwarded_indices = None + forwarded_counts = None + + def causal_conv1d_update_cpu(**kwargs): + nonlocal forwarded_counts, forwarded_indices + forwarded_indices = kwargs["conv_state_indices"] + forwarded_counts = kwargs["num_accepted_tokens"] + return kwargs["x"] + + monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True) + monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False) + monkeypatch.setattr( + gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu + ) + monkeypatch.setattr( + gdn_attention.ops, + "fused_sigmoid_gating_delta_rule_update_spec_cpu", + lambda **kwargs: kwargs["q"], + ) + + layer = types.SimpleNamespace( + activation="silu", + conv1d=types.SimpleNamespace(weight=torch.empty(1, CONV_KERNEL), bias=None), + A_log=None, + dt_bias=None, + rearrange_mixed_qkv=lambda x: (x.unsqueeze(0),) * 3, + ) + gdn_attention._spec_forward( + layer=layer, + attn_metadata_i=metadata, + mixed_qkv_spec=torch.zeros(8, 1, dtype=torch.bfloat16), + b_spec=torch.empty(0), + a_spec=torch.empty(0), + conv_buf=torch.empty(0), + ssm_state=torch.empty(0), + width=CONV_KERNEL, + state_len=0, + ) + + expected = ( + (forwarded_indices, torch.tensor([0, 4], dtype=torch.int32)), + (forwarded_counts, torch.tensor([1, 4], dtype=torch.int32)), + ) + for actual, reference in expected: + assert actual is not None + assert actual.is_contiguous() + assert actual.dtype == torch.int32 + torch.testing.assert_close(actual, reference) + + @pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS) @torch.inference_mode() def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> None: @@ -475,7 +678,182 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No @pytest.mark.skipif( not torch.cpu._is_amx_tile_supported(), - reason="causal_conv1d_fwd_cpu requires AMX/AVX512", + reason="requires AMX support", +) +@torch.inference_mode() +def test_causal_conv1d_update_cpu_accepts_wide_state() -> None: + state_len = CONV_KERNEL - 1 + wide_state_len = state_len + 5 + batch_size = 3 + is_vnni = True + x, weight, bias = _conv_inputs(batch_size) + conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32) + + narrow_state = _sd_conv_states(batch_size, state_len) + narrow_state.copy_( + tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state) + ) + wide_state = _sd_conv_states(batch_size, wide_state_len) + wide_state[:, :, :state_len].copy_(narrow_state) + wide_state[:, :, state_len:].fill_(7) + wide_tail = wide_state[:, :, state_len:].clone() + + conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + out_narrow = ops.causal_conv1d_update_cpu( + x=x, + conv_states=narrow_state, + weight=conv_weight, + bias=bias, + silu_activation=True, + conv_state_indices=conv_state_indices, + is_vnni=is_vnni, + ) + out_wide = ops.causal_conv1d_update_cpu( + x=x, + conv_states=wide_state, + weight=conv_weight, + bias=bias, + silu_activation=True, + conv_state_indices=conv_state_indices, + is_vnni=is_vnni, + ) + + torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2) + torch.testing.assert_close( + wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0 + ) + torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0) + + +def _ref_causal_conv1d_update_cpu_multi( + x: torch.Tensor, + conv_states: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + silu_activation: bool, + conv_state_indices: torch.Tensor, + num_accepted_tokens: torch.Tensor, +) -> torch.Tensor: + batch_size, seq_len, dim = x.shape + state_len = conv_states.size(2) + conv_out = torch.empty_like(x) + conv_weight = weight.unsqueeze(1) + + for i in range(batch_size): + slot = int(conv_state_indices[i].item()) + offset = int(num_accepted_tokens[i].item()) - 1 + state = conv_states[slot] + x_seq = x[i].transpose(0, 1).to(state.dtype) + prior = state[:, offset : offset + CONV_KERNEL - 1] + conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) + out = F.conv1d(conv_in, conv_weight, bias, groups=dim)[0] + if silu_activation: + out = F.silu(out) + conv_out[i] = out.transpose(0, 1).to(conv_out.dtype) + keep = state[:, offset + 1 : offset + 1 + (state_len - seq_len)] + state.copy_(torch.cat([keep, x_seq], dim=-1)) + + return conv_out + + +@pytest.mark.skipif( + not torch.cpu._is_amx_tile_supported(), + reason="requires AMX support", +) +@pytest.mark.parametrize( + ("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"), + [ + (1, 1, [1], False, False, False), + (1, 1, [1], True, True, True), + (4, 4, [1, 2, 3, 4], False, True, False), + (4, 4, [4, 3, 2, 1], True, False, True), + (4, 16, [1, 5, 10, 16], False, False, True), + (4, 16, [16, 10, 5, 1], True, True, False), + ], +) +@torch.inference_mode() +def test_causal_conv1d_update_cpu_multi_token_matches_python( + batch_size: int, + seq_len: int, + accepted_counts: list[int], + has_bias: bool, + silu_activation: bool, + is_vnni: bool, +) -> None: + dim = 96 + state_len = seq_len + 2 + x = tensor_cache(batch_size * seq_len * dim, torch.bfloat16).view( + batch_size, seq_len, dim + ) + weight = tensor_cache(dim * CONV_KERNEL, torch.bfloat16).view(dim, CONV_KERNEL) + bias = tensor_cache(dim, torch.bfloat16) if has_bias else None + conv_state_indices = torch.arange(batch_size - 1, -1, -1, dtype=torch.int32) + num_accepted_tokens = torch.tensor(accepted_counts, dtype=torch.int32) + + conv_states_ref = _sd_conv_states(batch_size, state_len, dim) + conv_states_ref.copy_( + tensor_cache(conv_states_ref.numel(), torch.bfloat16).view_as(conv_states_ref) + ) + conv_states = conv_states_ref.clone() + + conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + out = ops.causal_conv1d_update_cpu( + x=x, + conv_states=conv_states, + weight=conv_weight, + bias=bias, + silu_activation=silu_activation, + conv_state_indices=conv_state_indices, + is_vnni=is_vnni, + num_accepted_tokens=num_accepted_tokens, + ) + ref_out = _ref_causal_conv1d_update_cpu_multi( + x=x, + conv_states=conv_states_ref, + weight=weight, + bias=bias, + silu_activation=silu_activation, + conv_state_indices=conv_state_indices, + num_accepted_tokens=num_accepted_tokens, + ) + + torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(conv_states, conv_states_ref, atol=0, rtol=0) + + +@pytest.mark.skipif( + not torch.cpu._is_amx_tile_supported(), + reason="requires AMX support", +) +@pytest.mark.parametrize("num_accepted", [0, 17]) +@torch.inference_mode() +def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count( + num_accepted: int, +) -> None: + batch_size = 1 + seq_len = 16 + dim = 96 + state_len = seq_len + 2 + x = torch.zeros(batch_size, seq_len, dim, dtype=torch.bfloat16) + weight = torch.zeros(dim, CONV_KERNEL, dtype=torch.bfloat16) + conv_states = _sd_conv_states(batch_size, state_len, dim) + + with pytest.raises(RuntimeError, match="num_accepted_tokens must be in.*seqlen"): + ops.causal_conv1d_update_cpu( + x=x, + conv_states=conv_states, + weight=weight, + bias=None, + silu_activation=True, + conv_state_indices=torch.tensor([0], dtype=torch.int32), + is_vnni=False, + num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32), + ) + + +@pytest.mark.skipif( + not torch.cpu._is_amx_tile_supported(), + reason="requires AMX support", ) @pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS) @torch.inference_mode() @@ -515,6 +893,62 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) -> torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2) +@pytest.mark.skipif( + not torch.cpu._is_amx_tile_supported(), + reason="requires AMX support", +) +@torch.inference_mode() +def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None: + state_len = CONV_KERNEL - 1 + wide_state_len = state_len + 5 + is_vnni = True + seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5] + total_tokens = sum(seq_lens) + x, weight, bias = _conv_inputs(total_tokens) + query_start_loc = torch.tensor([0, seq_lens[0], total_tokens], dtype=torch.int32) + cache_indices = torch.tensor([2, 0], dtype=torch.int32) + has_initial_state = torch.tensor([True, False]) + + narrow_state = _sd_conv_states(3, state_len) + narrow_state.copy_( + tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state) + ) + wide_state = _sd_conv_states(3, wide_state_len) + wide_state[:, :, :state_len].copy_(narrow_state) + wide_state[:, :, state_len:].fill_(7) + wide_tail = wide_state[:, :, state_len:].clone() + + conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + out_narrow = ops.causal_conv1d_fwd_cpu( + x=x.transpose(0, 1), + weight=conv_weight, + bias=bias, + conv_states=narrow_state, + query_start_loc=query_start_loc, + cache_indices=cache_indices, + has_initial_state=has_initial_state, + silu_activation=True, + is_vnni=is_vnni, + ) + out_wide = ops.causal_conv1d_fwd_cpu( + x=x.transpose(0, 1), + weight=conv_weight, + bias=bias, + conv_states=wide_state, + query_start_loc=query_start_loc, + cache_indices=cache_indices, + has_initial_state=has_initial_state, + silu_activation=True, + is_vnni=is_vnni, + ) + + torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2) + torch.testing.assert_close( + wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0 + ) + torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0) + + @torch.inference_mode() def test_batch_memcpy_cpu_fallback() -> None: """The ctypes batch_memcpy fallback (used when triton-cpu is absent) must diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 82044c87c1..07a3a583f9 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3505,6 +3505,7 @@ def causal_conv1d_update_cpu( silu_activation: bool, conv_state_indices: torch.Tensor | None, is_vnni: bool, + num_accepted_tokens: torch.Tensor | None = None, ) -> torch.Tensor: return torch.ops._C.causal_conv1d_update_cpu( x, @@ -3512,7 +3513,7 @@ def causal_conv1d_update_cpu( weight, bias, silu_activation, - None, + num_accepted_tokens, conv_state_indices, -1, is_vnni, diff --git a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py index f93d2e18db..b20561c825 100644 --- a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py +++ b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py @@ -306,25 +306,21 @@ def _cpu_gdn_attention_spec_aware( width: int, state_len: int, ) -> None: - mixed_qkv = mixed_qkv.contiguous() - a = a.contiguous() - b = b.contiguous() - spec_sequence_masks = attn_metadata_i.spec_sequence_masks conv_buf = _conv_buffer_view(layer) # (num_slots, dim, state_len) ssm_state = _ssm_state_view(layer) if spec_sequence_masks is None: # No spec sequences in this batch (e.g. the prompt prefill step while - # speculative decoding is configured). Process as prefill/decode using - # torch conv (which only touches the first ``width-1`` columns of the - # wide buffer, leaving the rolling history untouched). + # speculative decoding is configured). Process as ordinary + # prefill/decode while touching only the first ``width-1`` columns of + # the wide buffer, leaving the rolling history untouched. _spec_aware_nonspec( layer, attn_metadata_i, - mixed_qkv, - b, - a, + mixed_qkv.contiguous(), + b.contiguous(), + a.contiguous(), core_attn_out, conv_buf, ssm_state, @@ -339,9 +335,9 @@ def _cpu_gdn_attention_spec_aware( num_decodes = attn_metadata_i.num_decodes if num_prefills == 0 and num_decodes == 0: - mixed_qkv_spec = mixed_qkv - b_spec = b - a_spec = a + mixed_qkv_spec = mixed_qkv.contiguous() + b_spec = b.contiguous() + a_spec = a.contiguous() spec_out_indx = None else: assert spec_token_indx is not None @@ -405,50 +401,71 @@ def _spec_forward( assert spec_qsl is not None assert num_accepted is not None - spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1].to("cpu", torch.int64) - num_acc_cpu = num_accepted[:num_spec_decodes].to("cpu", torch.int64) + spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1] seq_starts = spec_qsl_cpu[:-1] seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1] # ---- 1. Convolution (per-sequence rolling buffer) ---- - w2d = _unpacked_conv_weight(layer) # (dim, width) - dim = w2d.size(0) - w = w2d.unsqueeze(1) # (dim, 1, width) for F.conv1d depthwise + dim = mixed_qkv_spec.size(-1) bias = layer.conv1d.bias silu = layer.activation == "silu" - conv_out = torch.empty_like(mixed_qkv_spec) - col0 = spec_state_indices[:, 0].to("cpu", torch.int64) - for i in range(num_spec_decodes): - q_i = int(seq_lens[i].item()) - if q_i == 0: - continue - start = int(seq_starts[i].item()) - slot0 = int(col0[i].item()) - a_prev = int(num_acc_cpu[i].item()) - offset = a_prev - 1 - B = conv_buf[slot0] # (dim, state_len) - x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype) - prior = B[:, offset : offset + (width - 1)] - conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) # (1, dim, w-1+q) - out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i) - if silu: - out = F.silu(out) - conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype) - # Roll the buffer: drop ``a_prev`` from the front, append the new - # draft tokens, keep total length == state_len. - keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)] - new_B = torch.cat([keep, x_seq], dim=-1) - B.copy_(new_B) + can_use_native_conv = ( + torch.cpu._is_amx_tile_supported() + and not is_conv_state_dim_first() + and width == 4 + and num_spec_decodes > 0 + and bool(torch.all(seq_lens == seq_lens[0]).item()) + and int(seq_lens[0].item()) > 0 + ) + if can_use_native_conv: + q_i = int(seq_lens[0].item()) + conv_out = ops.causal_conv1d_update_cpu( + x=mixed_qkv_spec.view(num_spec_decodes, q_i, dim), + conv_states=conv_buf, + weight=layer.conv1d.weight, + bias=bias, + silu_activation=silu, + conv_state_indices=spec_state_indices[:num_spec_decodes, 0] + .to("cpu", torch.int32) + .contiguous(), + is_vnni=True, + num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32), + ).view_as(mixed_qkv_spec) + else: + w = _unpacked_conv_weight(layer).unsqueeze(1) + col0 = spec_state_indices[:num_spec_decodes, 0] + num_acc_cpu = num_accepted[:num_spec_decodes] + conv_out = torch.empty_like(mixed_qkv_spec) + for i in range(num_spec_decodes): + q_i = int(seq_lens[i].item()) + if q_i == 0: + continue + start = int(seq_starts[i].item()) + slot0 = int(col0[i].item()) + offset = int(num_acc_cpu[i].item()) - 1 + B = conv_buf[slot0] # (dim, state_len) + x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype) + prior = B[:, offset : offset + (width - 1)] + conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) + out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i) + if silu: + out = F.silu(out) + conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype) + # Roll the buffer: drop the accepted history from the front, append + # the new draft tokens, keep total length == state_len. + keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)] + new_B = torch.cat([keep, x_seq], dim=-1) + B.copy_(new_B) # ---- 2. Recurrent (multi-slot SSM state) ---- # Single fused kernel call: it runs the recurrence over each sequence's # draft tokens internally, resumes from slot ``num_accepted-1`` and stores # the state after token ``t`` into slot ``t`` (rollback for the next step). query, key, value = layer.rearrange_mixed_qkv(conv_out) - query = query.squeeze(0).contiguous() - key = key.squeeze(0).contiguous() - value = value.squeeze(0).contiguous() + query = query.squeeze(0) + key = key.squeeze(0) + value = value.squeeze(0) spec_idx = spec_state_indices[:num_spec_decodes].to(torch.int32).contiguous() num_acc = num_accepted[:num_spec_decodes].to(torch.int32).contiguous() cu = spec_qsl[: num_spec_decodes + 1].to(torch.int32).contiguous() @@ -458,8 +475,8 @@ def _spec_forward( q=query, k=key, v=value, - a=a_spec.contiguous(), - b=b_spec.contiguous(), + a=a_spec, + b=b_spec, initial_state_source=ssm_state, spec_state_indices=spec_idx, num_accepted_tokens=num_acc, @@ -469,15 +486,6 @@ def _spec_forward( return out_spec -def core_attn_out_like(layer, mixed_qkv_spec: torch.Tensor) -> torch.Tensor: - num_tokens = mixed_qkv_spec.size(0) - return torch.zeros( - (num_tokens, layer.num_v_heads // layer.tp_size, layer.head_v_dim), - dtype=mixed_qkv_spec.dtype, - device=mixed_qkv_spec.device, - ) - - def _spec_aware_nonspec( layer, attn_metadata_i: GDNAttentionMetadata, @@ -489,13 +497,19 @@ def _spec_aware_nonspec( ssm_state: torch.Tensor, width: int, ) -> None: - """Non-spec prefill/decode with a wide conv buffer (torch path).""" + """Non-spec prefill/decode with a wide conv buffer.""" state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor query_start_loc = attn_metadata_i.non_spec_query_start_loc assert state_indices_tensor is not None assert query_start_loc is not None + state_indices_tensor = state_indices_tensor.contiguous() - conv_weights = _unpacked_conv_weight(layer) + is_amx = torch.cpu._is_amx_tile_supported() + if is_amx and is_conv_state_dim_first(): + raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.") + + if not is_amx: + conv_weights = _unpacked_conv_weight(layer) num_decodes = attn_metadata_i.num_decodes num_decode_tokens = attn_metadata_i.num_decode_tokens @@ -507,42 +521,48 @@ def _spec_aware_nonspec( decode_b = b[:num_decode_tokens] decode_a = a[:num_decode_tokens] decode_state_indices = state_indices_tensor[:num_decodes] - # Only the first ``width-1`` columns hold the real conv state. - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - conv_state_view = conv_buf[:, :, : width - 1] - decode_conv_state = conv_state_view[decode_state_indices].contiguous() - decode_mixed_qkv = causal_conv1d_update_torch( - x=decode_mixed_qkv.unsqueeze(-1), - conv_state=decode_conv_state, - weight=conv_weights, - bias=layer.conv1d.bias, - activation=layer.activation, - ).squeeze(-1) - conv_state_view[decode_state_indices] = decode_conv_state - else: - decode_mixed_qkv = causal_conv1d_update_cpu( + if is_amx: + decode_mixed_qkv = ops.causal_conv1d_update_cpu( x=decode_mixed_qkv, - conv_state=conv_buf[:, :, : width - 1], - weight=conv_weights, + conv_states=conv_buf, + weight=layer.conv1d.weight, bias=layer.conv1d.bias, - activation=layer.activation, + silu_activation=layer.activation == "silu", conv_state_indices=decode_state_indices, + is_vnni=True, ) + else: + # Only the first ``width-1`` columns hold the real conv state. + conv_state_view = conv_buf[:, :, : width - 1] + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + decode_conv_state = conv_state_view[decode_state_indices].contiguous() + decode_mixed_qkv = causal_conv1d_update_torch( + x=decode_mixed_qkv.unsqueeze(-1), + conv_state=decode_conv_state, + weight=conv_weights, + bias=layer.conv1d.bias, + activation=layer.activation, + ).squeeze(-1) + conv_state_view[decode_state_indices] = decode_conv_state + else: + decode_mixed_qkv = causal_conv1d_update_cpu( + x=decode_mixed_qkv, + conv_state=conv_state_view, + weight=conv_weights, + bias=layer.conv1d.bias, + activation=layer.activation, + conv_state_indices=decode_state_indices, + ) query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv) - # rearrange_mixed_qkv can return views whose last dim is not - # contiguous; the fused CPU kernel requires a contiguous last dim. - query = query.contiguous() - key = key.contiguous() - value = value.contiguous() attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu( A_log=layer.A_log, dt_bias=layer.dt_bias, q=query, k=key, v=value, - a=decode_a.contiguous(), - b=decode_b.contiguous(), + a=decode_a, + b=decode_b, initial_state_source=ssm_state, initial_state_indices=decode_state_indices, cu_seqlens=query_start_loc[: num_decodes + 1], @@ -568,17 +588,29 @@ def _spec_aware_nonspec( prefill_has_initial_state = has_initial_state[ num_decodes : num_decodes + num_prefills ] - # ``causal_conv1d_torch`` only touches columns [:width-1] of the buffer. - prefill_mixed_qkv = causal_conv1d_torch( - x=prefill_mixed_qkv.transpose(0, 1), - weight=conv_weights, - bias=layer.conv1d.bias, - conv_states=conv_buf, - query_start_loc=prefill_query_start_loc, - cache_indices=prefill_state_indices, - has_initial_state=prefill_has_initial_state, - activation=layer.activation, - ).transpose(0, 1) + if is_amx: + prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu( + x=prefill_mixed_qkv.transpose(0, 1), + weight=layer.conv1d.weight, + bias=layer.conv1d.bias, + conv_states=conv_buf, + query_start_loc=prefill_query_start_loc, + cache_indices=prefill_state_indices, + has_initial_state=prefill_has_initial_state, + silu_activation=layer.activation == "silu", + is_vnni=True, + ).transpose(0, 1) + else: + prefill_mixed_qkv = causal_conv1d_torch( + x=prefill_mixed_qkv.transpose(0, 1), + weight=conv_weights, + bias=layer.conv1d.bias, + conv_states=conv_buf, + query_start_loc=prefill_query_start_loc, + cache_indices=prefill_state_indices, + has_initial_state=prefill_has_initial_state, + activation=layer.activation, + ).transpose(0, 1) query, key, value = layer.rearrange_mixed_qkv(prefill_mixed_qkv) g, beta = ops.fused_gdn_gating_cpu( @@ -618,24 +650,41 @@ def _spec_aware_nonspec_subset( Returns outputs ordered like ``non_spec_token_indx``. """ - out = core_attn_out_like(layer, mixed_qkv) has_initial_state = attn_metadata_i.has_initial_state prefill_state_indices = attn_metadata_i.prefill_state_indices prefill_qsl = attn_metadata_i.prefill_query_start_loc assert prefill_state_indices is not None and prefill_qsl is not None assert has_initial_state is not None + prefill_state_indices = prefill_state_indices.contiguous() - conv_weights = _unpacked_conv_weight(layer) - conv_out = causal_conv1d_torch( - x=mixed_qkv.transpose(0, 1), - weight=conv_weights, - bias=layer.conv1d.bias, - conv_states=conv_buf, - query_start_loc=prefill_qsl, - cache_indices=prefill_state_indices, - has_initial_state=has_initial_state, - activation=layer.activation, - ).transpose(0, 1) + is_amx = torch.cpu._is_amx_tile_supported() + if is_amx and is_conv_state_dim_first(): + raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.") + + if is_amx: + conv_out = ops.causal_conv1d_fwd_cpu( + x=mixed_qkv.transpose(0, 1), + weight=layer.conv1d.weight, + bias=layer.conv1d.bias, + conv_states=conv_buf, + query_start_loc=prefill_qsl, + cache_indices=prefill_state_indices, + has_initial_state=has_initial_state, + silu_activation=layer.activation == "silu", + is_vnni=True, + ).transpose(0, 1) + else: + conv_weights = _unpacked_conv_weight(layer) + conv_out = causal_conv1d_torch( + x=mixed_qkv.transpose(0, 1), + weight=conv_weights, + bias=layer.conv1d.bias, + conv_states=conv_buf, + query_start_loc=prefill_qsl, + cache_indices=prefill_state_indices, + has_initial_state=has_initial_state, + activation=layer.activation, + ).transpose(0, 1) query, key, value = layer.rearrange_mixed_qkv(conv_out) g, beta = ops.fused_gdn_gating_cpu( @@ -658,8 +707,7 @@ def _spec_aware_nonspec_subset( ssm_state[prefill_state_indices] = last_recurrent_state.to( ssm_state.dtype, copy=False ) - out[:] = attn_out.squeeze(0) - return out + return attn_out.squeeze(0) def cpu_gdn_attention_core_fake(