[CPU] Add fused GDN support for AMX CPU platform (#42707)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-05-18 03:04:36 -07:00
committed by GitHub
parent 2267f70070
commit b4601ad43f
8 changed files with 407 additions and 14 deletions
+129
View File
@@ -3370,6 +3370,135 @@ def fp8_scaled_mm_cpu(
)
def chunk_gated_delta_rule_cpu(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
initial_state: torch.Tensor,
output_final_state: bool,
cu_seqlens: torch.Tensor,
head_first: bool,
use_qk_l2norm_in_kernel: bool,
eps: float = 1e-5,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops._C.chunk_gated_delta_rule_cpu(
query,
key,
value,
g,
beta,
initial_state,
output_final_state,
cu_seqlens,
head_first,
use_qk_l2norm_in_kernel,
eps,
)
def fused_sigmoid_gating_delta_rule_update_cpu(
A_log: torch.Tensor,
dt_bias: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
initial_state_source: torch.Tensor,
initial_state_indices: torch.Tensor,
cu_seqlens: torch.Tensor,
use_qk_l2norm_in_kernel: bool,
softplus_beta: float = 1.0,
softplus_threshold: float = 20.0,
) -> torch.Tensor:
return torch.ops._C.fused_sigmoid_gating_delta_rule_update_cpu(
A_log,
dt_bias,
q,
k,
v,
a,
b,
initial_state_source,
initial_state_indices,
cu_seqlens,
use_qk_l2norm_in_kernel,
softplus_beta,
softplus_threshold,
)
def fused_gdn_gating_cpu(
A_log: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
dt_bias: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops._C.fused_gdn_gating_cpu(
A_log,
a,
b,
dt_bias,
)
def causal_conv1d_weight_pack(
weight: torch.Tensor,
) -> torch.Tensor:
return torch.ops._C.causal_conv1d_weight_pack(
weight,
)
def causal_conv1d_fwd_cpu(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
conv_states: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
cache_indices: torch.Tensor | None,
has_initial_state: torch.Tensor | None,
silu_activation: bool,
is_vnni: bool,
) -> torch.Tensor:
return torch.ops._C.causal_conv1d_fwd_cpu(
x,
weight,
bias,
conv_states,
query_start_loc,
cache_indices,
has_initial_state,
silu_activation,
-1,
is_vnni,
)
def causal_conv1d_update_cpu(
x: torch.Tensor,
conv_states: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.ops._C.causal_conv1d_update_cpu(
x,
conv_states,
weight,
bias,
silu_activation,
None,
conv_state_indices,
-1,
is_vnni,
)
class CPUDNNLGEMMHandler:
def __init__(self) -> None:
self.handler_tensor: torch.Tensor | None = None
-3
View File
@@ -213,9 +213,6 @@ class UnquantizedLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if current_platform.is_cpu():
if layer.weight.ndim != 2:
# this is not a linear layer
return
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
@@ -5,6 +5,7 @@ from __future__ import annotations
import torch
import vllm._custom_ops as ops
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
@@ -55,6 +56,16 @@ def cpu_gdn_attention_core(
and attn_metadata_i.num_accepted_tokens is None
), "speculative decode not supported in CPU GDN attention."
if torch.cpu._is_amx_tile_supported():
return cpu_gdn_attention_core_amx(
mixed_qkv,
b,
a,
core_attn_out,
attn_metadata_i,
layer,
)
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
query_start_loc = attn_metadata_i.non_spec_query_start_loc
assert state_indices_tensor is not None
@@ -194,6 +205,135 @@ def cpu_gdn_attention_core_fake(
return
def cpu_gdn_attention_core_amx(
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
core_attn_out: torch.Tensor,
attn_metadata_i: GDNAttentionMetadata,
layer: torch.nn.Module,
):
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
query_start_loc = attn_metadata_i.non_spec_query_start_loc
assert state_indices_tensor is not None
assert query_start_loc is not None
# [num_allocated_slots, kernel - 1, conv_dim]
conv_state = layer.kv_cache[0]
if is_conv_state_dim_first():
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
# reshape to [num_allocated_slots, conv_dim, kernel - 1]
conv_state_t = conv_state.transpose(1, 2)
# [num_allocated_slots, num_v_heads / tp_size, v_dim, k_dim]
ssm_state = layer.kv_cache[1]
# rehape to [num_allocated_slots, num_v_heads / tp_size, k_dim, v_dim]
num_allocated_slots, head_num, v_dim, k_dim = ssm_state.size()
ssm_state = ssm_state.view(
num_allocated_slots,
head_num,
k_dim,
v_dim,
)
mixed_qkv = mixed_qkv.contiguous()
a = a.contiguous()
b = b.contiguous()
num_decodes = attn_metadata_i.num_decodes
num_decode_tokens = attn_metadata_i.num_decode_tokens
num_prefills = attn_metadata_i.num_prefills
num_prefill_tokens = attn_metadata_i.num_prefill_tokens
if num_decodes > 0:
decode_mixed_qkv = mixed_qkv[:num_decode_tokens]
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
decode_mixed_qkv = ops.causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_states=conv_state_t,
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
silu_activation=layer.activation == "silu",
conv_state_indices=decode_state_indices,
is_vnni=True,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
A_log=layer.A_log,
dt_bias=layer.dt_bias,
q=query,
k=key,
v=value,
a=decode_a,
b=decode_b,
initial_state_source=ssm_state,
initial_state_indices=decode_state_indices,
cu_seqlens=query_start_loc[: num_decodes + 1],
use_qk_l2norm_in_kernel=True,
)
core_attn_out[:num_decode_tokens] = attn_out.squeeze(1)
if num_prefills > 0:
has_initial_state = attn_metadata_i.has_initial_state
assert has_initial_state is not None
prefill_token_start = num_decode_tokens
prefill_token_end = prefill_token_start + num_prefill_tokens
prefill_mixed_qkv = mixed_qkv[prefill_token_start:prefill_token_end]
prefill_b = b[prefill_token_start:prefill_token_end]
prefill_a = a[prefill_token_start:prefill_token_end]
prefill_state_indices = state_indices_tensor[
num_decodes : num_decodes + num_prefills
]
prefill_query_start_loc = (
query_start_loc[num_decodes : num_decodes + num_prefills + 1]
- num_decode_tokens
)
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
conv_states=conv_state_t,
query_start_loc=prefill_query_start_loc,
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
).transpose(0, 1)
query, key, value = layer.rearrange_mixed_qkv(prefill_mixed_qkv)
g, beta = ops.fused_gdn_gating_cpu(
A_log=layer.A_log, a=prefill_a, b=prefill_b, dt_bias=layer.dt_bias
)
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
attn_out, last_recurrent_state = ops.chunk_gated_delta_rule_cpu(
query=query,
key=key,
value=value,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=prefill_query_start_loc,
head_first=False,
use_qk_l2norm_in_kernel=True,
)
ssm_state[prefill_state_indices] = last_recurrent_state.to(
ssm_state.dtype, copy=False
)
core_attn_out[prefill_token_start:prefill_token_end] = attn_out.squeeze(0)
def register_cpu_gdn_attention_ops() -> None:
global _CPU_GDN_ATTENTION_OPS_REGISTERED
if _CPU_GDN_ATTENTION_OPS_REGISTERED:
+13
View File
@@ -233,6 +233,19 @@ def dispatch_cpu_unquantized_gemm(
layer.cpu_linear = torch.nn.functional.linear
return
if layer.weight.ndim != 2:
# this is not a linear layer
# For now it should be a causal_conv1d op
if torch.cpu._is_amx_tile_supported():
# prepack conv weight
layer.weight.data = ops.causal_conv1d_weight_pack(
layer.weight.view(
layer.weight.size(0),
layer.weight.size(2),
)
)
return
N, K = layer.weight.size()
dtype = layer.weight.dtype
+19
View File
@@ -139,6 +139,7 @@ class CpuPlatform(Platform):
# supported as it is not possible to set the OMP environment correctly
if parallel_config.distributed_executor_backend == "uni":
parallel_config.distributed_executor_backend = "mp"
if parallel_config.worker_cls == "auto":
parallel_config.worker_cls = "vllm.v1.worker.cpu_worker.CPUWorker"
# Disable DBO
@@ -146,6 +147,20 @@ class CpuPlatform(Platform):
logger.warning("Dual-Batch Overlap is not supported on CPU, disabled.")
parallel_config.enable_dbo = False
if torch.cpu._is_amx_tile_supported() and (
model_config is not None
and model_config.get_num_layers_by_block_type(
parallel_config, "linear_attention"
)
> 0
):
cache_config.enable_prefix_caching = False
scheduler_config.enable_chunked_prefill = False
logger.warning(
"Disabled unsupported prefix caching and chunked prefill "
"for linear attention on AMX CPU platforms."
)
# Note: workaround for v1 gpu_model_runner
from vllm.config import CompilationMode
@@ -211,6 +226,10 @@ class CpuPlatform(Platform):
# Avoid inductor generates num_thread() and breaks the thread binding
os.environ["TORCHINDUCTOR_CPP_DYNAMIC_THREADS"] = "1"
# For efficient conv state memory access
if torch.cpu._is_amx_tile_supported():
os.environ["VLLM_SSM_CONV_STATE_LAYOUT"] = "SD"
ld_preload_str = os.getenv("LD_PRELOAD", "")
cpu_architecture = Platform.get_cpu_architecture()