[Bugfix][CPU] Enable C++ causal_conv1d GDN path and float32 SSM cache on non-AMX AVX-512BF16 CPUs (#49688)

Signed-off-by: Dinesh Chitlangia <dineshchitlangia@gmail.com>
This commit is contained in:
Dinesh Chitlangia
2026-08-19 10:50:59 -05:00
committed by GitHub
parent 17dbd42930
commit 525b7bbb3a
4 changed files with 290 additions and 17 deletions
@@ -103,13 +103,15 @@ def _cpu_gdn_attention_nonspec(
assert state_indices_tensor is not None
assert query_start_loc is not None
is_amx = torch.cpu._is_amx_tile_supported()
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported()
conv_state = layer.kv_cache[0]
if is_amx:
# AMX causal conv requires [num_allocated_slots, kernel - 1, conv_dim].
if use_cpp_conv:
# C++ conv requires [num_allocated_slots, kernel - 1, conv_dim] (SD).
if is_conv_state_dim_first():
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
raise RuntimeError("C++ CPU GDN attention requires `SD` conv_state layout.")
conv_state = conv_state.transpose(1, 2)
else:
if not is_conv_state_dim_first():
@@ -138,7 +140,7 @@ def _cpu_gdn_attention_nonspec(
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
if is_amx:
if use_cpp_conv:
decode_mixed_qkv = ops.causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_states=conv_state,
@@ -207,7 +209,7 @@ def _cpu_gdn_attention_nonspec(
num_decodes : num_decodes + num_prefills
]
if is_amx:
if use_cpp_conv:
prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
+5 -2
View File
@@ -280,7 +280,10 @@ def dispatch_cpu_unquantized_gemm(
if layer.weight.ndim != 2:
# this is not a linear layer
# For now it should be a causal_conv1d op or MoE 3D expert weights
if torch.cpu._is_amx_tile_supported() and hasattr(
# The C++ causal_conv1d kernels use VDPBF16PS (no AMX tiles), so the
# VNNI weight prepack applies to any AVX-512BF16 CPU, not just AMX
# (e.g. AMD Zen5/Turin).
if torch.cpu._is_avx512_bf16_supported() and hasattr(
ops, "causal_conv1d_weight_pack"
):
# prepack conv weight
@@ -293,7 +296,7 @@ def dispatch_cpu_unquantized_gemm(
.clone()
)
# Stash the un-packed (dim, width) weight so the speculative-decode
# GDN path (which uses torch conv, not the AMX kernel) can use it.
# GDN path (which uses torch conv, not the C++ kernel) can use it.
layer._cpu_unpacked_conv_weight = unpacked
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
return