[CPU][Spec Decode] Support DFlash speculative decoding for GDN models on CPU (#46090)

Signed-off-by: guybd <guy.boudoukh@intel.com>
Signed-off-by: Guy Boudoukh <guy.boudoukh@intel.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
guybd
2026-07-13 04:16:18 +00:00
committed by GitHub
co-authored by Cursor Li, Jiang <jiang1.li@intel.com>
parent d973cce3ca
commit 05fa8183a6
5 changed files with 755 additions and 10 deletions
@@ -4,6 +4,7 @@
from __future__ import annotations
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from vllm.forward_context import ForwardContext, get_forward_context
@@ -46,11 +47,52 @@ def cpu_gdn_attention_core(
if attn_metadata_i.num_actual_tokens == 0:
return
assert mixed_qkv.dtype == torch.bfloat16, "CPU GDN attention requires BF16."
# The conv-state cache temporal dim is ``width - 1`` when speculative
# decoding is disabled, and ``width - 1 + num_spec`` when it is enabled
# (vLLM allocates a wider rolling buffer to support draft-token rollback).
width = layer.conv1d.weight.size(-1)
conv_cache = layer.kv_cache[0]
if is_conv_state_dim_first():
# DS layout: (num_slots, dim, state_len)
state_len = conv_cache.size(-1)
else:
# SD layout: (num_slots, state_len, dim)
state_len = conv_cache.size(-2)
spec_decode_cache = state_len > (width - 1)
if not spec_decode_cache:
# ===== Fast path: speculative decoding NOT configured =====
# conv-state is exactly (..., dim, width-1); use the original AMX /
# torch implementation unchanged.
_cpu_gdn_attention_nonspec(
layer, attn_metadata_i, mixed_qkv, b, a, core_attn_out
)
return
# ===== Speculative-decoding-aware path (wide rolling conv buffer) =====
_cpu_gdn_attention_spec_aware(
layer, attn_metadata_i, mixed_qkv, b, a, core_attn_out, width, state_len
)
# ---------------------------------------------------------------------------
# Non-speculative implementation: conv-state is exactly width-1.
# ---------------------------------------------------------------------------
def _cpu_gdn_attention_nonspec(
layer,
attn_metadata_i: GDNAttentionMetadata,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
assert (
attn_metadata_i.spec_sequence_masks is None
and attn_metadata_i.num_accepted_tokens is None
), "speculative decode not supported in CPU GDN attention."
assert mixed_qkv.dtype == torch.bfloat16, "CPU GDN attention requires BF16."
), "speculative decode not supported without a wide conv-state cache."
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
query_start_loc = attn_metadata_i.non_spec_query_start_loc
@@ -79,12 +121,7 @@ def cpu_gdn_attention_core(
b = b.contiguous()
num_allocated_slots, head_num, v_dim, k_dim = ssm_state.size()
ssm_state = ssm_state.view(
num_allocated_slots,
head_num,
k_dim,
v_dim,
)
ssm_state = ssm_state.view(num_allocated_slots, head_num, k_dim, v_dim)
num_decodes = attn_metadata_i.num_decodes
num_decode_tokens = attn_metadata_i.num_decode_tokens
@@ -109,7 +146,6 @@ def cpu_gdn_attention_core(
)
else:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
# [B, dim] -> [B, dim, 1]
x=decode_mixed_qkv.unsqueeze(-1),
@@ -207,6 +243,403 @@ def cpu_gdn_attention_core(
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
# ---------------------------------------------------------------------------
# Speculative-decoding-aware implementation (wide rolling conv buffer).
#
# Mirrors the GPU `_forward_core` spec path using torch + the existing CPU
# kernels. The conv-state is a per-sequence rolling buffer of length
# ``state_len = width - 1 + num_spec`` (single cache slot, column 0 of the
# spec block table), and the SSM recurrent state is stored multi-slot: the
# state *after* draft token ``t`` lives in block-table column ``t``, so the
# next step can resume from the slot of the last accepted token.
# ---------------------------------------------------------------------------
def _conv_buffer_view(layer) -> torch.Tensor:
"""Return the conv-state cache as (num_slots, dim, state_len)."""
conv_cache = layer.kv_cache[0]
if is_conv_state_dim_first():
return conv_cache # (num_slots, dim, state_len)
return conv_cache.transpose(-1, -2) # view -> (num_slots, dim, state_len)
def _ssm_state_view(layer) -> torch.Tensor:
ssm_state = layer.kv_cache[1]
num_slots, head_num, v_dim, k_dim = ssm_state.size()
return ssm_state.view(num_slots, head_num, k_dim, v_dim)
def _unpacked_conv_weight(layer) -> torch.Tensor:
"""Return the plain (dim, width) conv weight.
On AMX the conv1d weight is VNNI-packed in place at load time (only usable
by the AMX C++ kernel), so the torch spec-decode path relies on the
un-packed copy stashed by ``dispatch_cpu_unquantized_gemm``.
"""
w = getattr(layer.conv1d, "_cpu_unpacked_conv_weight", None)
if w is not None:
return w
w = layer.conv1d.weight
if w.dim() == 3:
return w.view(w.size(0), w.size(2))
return w
def _cpu_gdn_attention_spec_aware(
layer,
attn_metadata_i: GDNAttentionMetadata,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
width: int,
state_len: int,
) -> None:
mixed_qkv = mixed_qkv.contiguous()
a = a.contiguous()
b = b.contiguous()
spec_sequence_masks = attn_metadata_i.spec_sequence_masks
conv_buf = _conv_buffer_view(layer) # (num_slots, dim, state_len)
ssm_state = _ssm_state_view(layer)
if spec_sequence_masks is None:
# No spec sequences in this batch (e.g. the prompt prefill step while
# speculative decoding is configured). Process as prefill/decode using
# torch conv (which only touches the first ``width-1`` columns of the
# wide buffer, leaving the rolling history untouched).
_spec_aware_nonspec(
layer,
attn_metadata_i,
mixed_qkv,
b,
a,
core_attn_out,
conv_buf,
ssm_state,
width,
)
return
# Split spec vs non-spec tokens.
spec_token_indx = attn_metadata_i.spec_token_indx
non_spec_token_indx = attn_metadata_i.non_spec_token_indx
num_prefills = attn_metadata_i.num_prefills
num_decodes = attn_metadata_i.num_decodes
if num_prefills == 0 and num_decodes == 0:
mixed_qkv_spec = mixed_qkv
b_spec = b
a_spec = a
spec_out_indx = None
else:
assert spec_token_indx is not None
mixed_qkv_spec = mixed_qkv.index_select(0, spec_token_indx)
b_spec = b.index_select(0, spec_token_indx)
a_spec = a.index_select(0, spec_token_indx)
spec_out_indx = spec_token_indx
spec_out = _spec_forward(
layer,
attn_metadata_i,
mixed_qkv_spec,
b_spec,
a_spec,
conv_buf,
ssm_state,
width,
state_len,
)
# Process the (rare) non-spec subset (prefill) tokens, if any.
nonspec_out = None
if (num_prefills > 0 or num_decodes > 0) and non_spec_token_indx is not None:
mixed_qkv_ns = mixed_qkv.index_select(0, non_spec_token_indx)
b_ns = b.index_select(0, non_spec_token_indx)
a_ns = a.index_select(0, non_spec_token_indx)
nonspec_out = _spec_aware_nonspec_subset(
layer, attn_metadata_i, mixed_qkv_ns, b_ns, a_ns, conv_buf, ssm_state, width
)
# Scatter outputs back to their token positions.
if spec_out_indx is None:
core_attn_out[: spec_out.size(0)] = spec_out
else:
core_attn_out.index_copy_(0, spec_out_indx, spec_out)
if nonspec_out is not None:
core_attn_out.index_copy_(0, non_spec_token_indx, nonspec_out)
def _spec_forward(
layer,
attn_metadata_i: GDNAttentionMetadata,
mixed_qkv_spec: torch.Tensor,
b_spec: torch.Tensor,
a_spec: torch.Tensor,
conv_buf: torch.Tensor,
ssm_state: torch.Tensor,
width: int,
state_len: int,
) -> torch.Tensor:
"""Run the GDN core for the multi-query (speculative) tokens.
Returns the core attention output for the spec tokens, in the same token
order as ``mixed_qkv_spec`` (i.e. ordered by ``spec_query_start_loc``).
"""
num_spec_decodes = attn_metadata_i.num_spec_decodes
spec_state_indices = attn_metadata_i.spec_state_indices_tensor
spec_qsl = attn_metadata_i.spec_query_start_loc
num_accepted = attn_metadata_i.num_accepted_tokens
assert spec_state_indices is not None
assert spec_qsl is not None
assert num_accepted is not None
spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1].to("cpu", torch.int64)
num_acc_cpu = num_accepted[:num_spec_decodes].to("cpu", torch.int64)
seq_starts = spec_qsl_cpu[:-1]
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
# ---- 1. Convolution (per-sequence rolling buffer) ----
w2d = _unpacked_conv_weight(layer) # (dim, width)
dim = w2d.size(0)
w = w2d.unsqueeze(1) # (dim, 1, width) for F.conv1d depthwise
bias = layer.conv1d.bias
silu = layer.activation == "silu"
conv_out = torch.empty_like(mixed_qkv_spec)
col0 = spec_state_indices[:, 0].to("cpu", torch.int64)
for i in range(num_spec_decodes):
q_i = int(seq_lens[i].item())
if q_i == 0:
continue
start = int(seq_starts[i].item())
slot0 = int(col0[i].item())
a_prev = int(num_acc_cpu[i].item())
offset = a_prev - 1
B = conv_buf[slot0] # (dim, state_len)
x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype)
prior = B[:, offset : offset + (width - 1)]
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) # (1, dim, w-1+q)
out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i)
if silu:
out = F.silu(out)
conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype)
# Roll the buffer: drop ``a_prev`` from the front, append the new
# draft tokens, keep total length == state_len.
keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)]
new_B = torch.cat([keep, x_seq], dim=-1)
B.copy_(new_B)
# ---- 2. Recurrent (multi-slot SSM state) ----
# Single fused kernel call: it runs the recurrence over each sequence's
# draft tokens internally, resumes from slot ``num_accepted-1`` and stores
# the state after token ``t`` into slot ``t`` (rollback for the next step).
query, key, value = layer.rearrange_mixed_qkv(conv_out)
query = query.squeeze(0).contiguous()
key = key.squeeze(0).contiguous()
value = value.squeeze(0).contiguous()
spec_idx = spec_state_indices[:num_spec_decodes].to(torch.int32).contiguous()
num_acc = num_accepted[:num_spec_decodes].to(torch.int32).contiguous()
cu = spec_qsl[: num_spec_decodes + 1].to(torch.int32).contiguous()
out_spec = ops.fused_sigmoid_gating_delta_rule_update_spec_cpu(
A_log=layer.A_log,
dt_bias=layer.dt_bias,
q=query,
k=key,
v=value,
a=a_spec.contiguous(),
b=b_spec.contiguous(),
initial_state_source=ssm_state,
spec_state_indices=spec_idx,
num_accepted_tokens=num_acc,
cu_seqlens=cu,
use_qk_l2norm_in_kernel=True,
)
return out_spec
def core_attn_out_like(layer, mixed_qkv_spec: torch.Tensor) -> torch.Tensor:
num_tokens = mixed_qkv_spec.size(0)
return torch.zeros(
(num_tokens, layer.num_v_heads // layer.tp_size, layer.head_v_dim),
dtype=mixed_qkv_spec.dtype,
device=mixed_qkv_spec.device,
)
def _spec_aware_nonspec(
layer,
attn_metadata_i: GDNAttentionMetadata,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
conv_buf: torch.Tensor,
ssm_state: torch.Tensor,
width: int,
) -> None:
"""Non-spec prefill/decode with a wide conv buffer (torch path)."""
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
query_start_loc = attn_metadata_i.non_spec_query_start_loc
assert state_indices_tensor is not None
assert query_start_loc is not None
conv_weights = _unpacked_conv_weight(layer)
num_decodes = attn_metadata_i.num_decodes
num_decode_tokens = attn_metadata_i.num_decode_tokens
num_prefills = attn_metadata_i.num_prefills
num_prefill_tokens = attn_metadata_i.num_prefill_tokens
if num_decodes > 0:
decode_mixed_qkv = mixed_qkv[:num_decode_tokens]
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
# Only the first ``width-1`` columns hold the real conv state.
decode_conv_state = conv_buf[decode_state_indices][
:, :, : width - 1
].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_buf[decode_state_indices, :, : width - 1] = decode_conv_state
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
# rearrange_mixed_qkv can return views whose last dim is not
# contiguous; the fused CPU kernel requires a contiguous last dim.
query = query.contiguous()
key = key.contiguous()
value = value.contiguous()
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
A_log=layer.A_log,
dt_bias=layer.dt_bias,
q=query,
k=key,
v=value,
a=decode_a.contiguous(),
b=decode_b.contiguous(),
initial_state_source=ssm_state,
initial_state_indices=decode_state_indices,
cu_seqlens=query_start_loc[: num_decodes + 1],
use_qk_l2norm_in_kernel=True,
)
core_attn_out[:num_decode_tokens] = attn_out.squeeze(1)
if num_prefills > 0:
has_initial_state = attn_metadata_i.has_initial_state
assert has_initial_state is not None
prefill_token_start = num_decode_tokens
prefill_token_end = prefill_token_start + num_prefill_tokens
prefill_mixed_qkv = mixed_qkv[prefill_token_start:prefill_token_end]
prefill_b = b[prefill_token_start:prefill_token_end]
prefill_a = a[prefill_token_start:prefill_token_end]
prefill_state_indices = state_indices_tensor[
num_decodes : num_decodes + num_prefills
]
prefill_query_start_loc = (
query_start_loc[num_decodes : num_decodes + num_prefills + 1]
- num_decode_tokens
)
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
# ``causal_conv1d_torch`` only touches columns [:width-1] of the buffer.
prefill_mixed_qkv = causal_conv1d_torch(
x=prefill_mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_query_start_loc,
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
activation=layer.activation,
).transpose(0, 1)
query, key, value = layer.rearrange_mixed_qkv(prefill_mixed_qkv)
g, beta = ops.fused_gdn_gating_cpu(
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=prefill_query_start_loc,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
)
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
def _spec_aware_nonspec_subset(
layer,
attn_metadata_i: GDNAttentionMetadata,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
conv_buf: torch.Tensor,
ssm_state: torch.Tensor,
width: int,
) -> torch.Tensor:
"""Process non-spec (prefill) tokens that coexist with spec sequences.
Returns outputs ordered like ``non_spec_token_indx``.
"""
out = core_attn_out_like(layer, mixed_qkv)
has_initial_state = attn_metadata_i.has_initial_state
prefill_state_indices = attn_metadata_i.prefill_state_indices
prefill_qsl = attn_metadata_i.prefill_query_start_loc
assert prefill_state_indices is not None and prefill_qsl is not None
assert has_initial_state is not None
conv_weights = _unpacked_conv_weight(layer)
conv_out = causal_conv1d_torch(
x=mixed_qkv.transpose(0, 1),
weight=conv_weights,
bias=layer.conv1d.bias,
conv_states=conv_buf,
query_start_loc=prefill_qsl,
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
activation=layer.activation,
).transpose(0, 1)
query, key, value = layer.rearrange_mixed_qkv(conv_out)
g, beta = ops.fused_gdn_gating_cpu(
A_log=layer.A_log, a=a, b=b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=prefill_qsl,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
)
out[:] = attn_out.squeeze(0)
return out
def cpu_gdn_attention_core_fake(
mixed_qkv: torch.Tensor,
b: torch.Tensor,
+7 -1
View File
@@ -239,12 +239,18 @@ def dispatch_cpu_unquantized_gemm(
# For now it should be a causal_conv1d op
if torch.cpu._is_amx_tile_supported():
# prepack conv weight
layer.weight.data = ops.causal_conv1d_weight_pack(
unpacked = (
layer.weight.view(
layer.weight.size(0),
layer.weight.size(2),
)
.contiguous()
.clone()
)
# Stash the un-packed (dim, width) weight so the speculative-decode
# GDN path (which uses torch conv, not the AMX kernel) can use it.
layer._cpu_unpacked_conv_weight = unpacked
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
return
N, K = layer.weight.size()