[Bugfix][CPU] Enable C++ causal_conv1d GDN path and float32 SSM cache on non-AMX AVX-512BF16 CPUs (#49688)
Signed-off-by: Dinesh Chitlangia <dineshchitlangia@gmail.com>
This commit is contained in:
@@ -103,13 +103,15 @@ def _cpu_gdn_attention_nonspec(
|
||||
assert state_indices_tensor is not None
|
||||
assert query_start_loc is not None
|
||||
|
||||
is_amx = torch.cpu._is_amx_tile_supported()
|
||||
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
|
||||
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
|
||||
use_cpp_conv = torch.cpu._is_avx512_bf16_supported()
|
||||
|
||||
conv_state = layer.kv_cache[0]
|
||||
if is_amx:
|
||||
# AMX causal conv requires [num_allocated_slots, kernel - 1, conv_dim].
|
||||
if use_cpp_conv:
|
||||
# C++ conv requires [num_allocated_slots, kernel - 1, conv_dim] (SD).
|
||||
if is_conv_state_dim_first():
|
||||
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
|
||||
raise RuntimeError("C++ CPU GDN attention requires `SD` conv_state layout.")
|
||||
conv_state = conv_state.transpose(1, 2)
|
||||
else:
|
||||
if not is_conv_state_dim_first():
|
||||
@@ -138,7 +140,7 @@ def _cpu_gdn_attention_nonspec(
|
||||
decode_b = b[:num_decode_tokens]
|
||||
decode_a = a[:num_decode_tokens]
|
||||
decode_state_indices = state_indices_tensor[:num_decodes]
|
||||
if is_amx:
|
||||
if use_cpp_conv:
|
||||
decode_mixed_qkv = ops.causal_conv1d_update_cpu(
|
||||
x=decode_mixed_qkv,
|
||||
conv_states=conv_state,
|
||||
@@ -207,7 +209,7 @@ def _cpu_gdn_attention_nonspec(
|
||||
num_decodes : num_decodes + num_prefills
|
||||
]
|
||||
|
||||
if is_amx:
|
||||
if use_cpp_conv:
|
||||
prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu(
|
||||
x=prefill_mixed_qkv.transpose(0, 1),
|
||||
weight=layer.conv1d.weight,
|
||||
|
||||
@@ -280,7 +280,10 @@ def dispatch_cpu_unquantized_gemm(
|
||||
if layer.weight.ndim != 2:
|
||||
# this is not a linear layer
|
||||
# For now it should be a causal_conv1d op or MoE 3D expert weights
|
||||
if torch.cpu._is_amx_tile_supported() and hasattr(
|
||||
# The C++ causal_conv1d kernels use VDPBF16PS (no AMX tiles), so the
|
||||
# VNNI weight prepack applies to any AVX-512BF16 CPU, not just AMX
|
||||
# (e.g. AMD Zen5/Turin).
|
||||
if torch.cpu._is_avx512_bf16_supported() and hasattr(
|
||||
ops, "causal_conv1d_weight_pack"
|
||||
):
|
||||
# prepack conv weight
|
||||
@@ -293,7 +296,7 @@ def dispatch_cpu_unquantized_gemm(
|
||||
.clone()
|
||||
)
|
||||
# Stash the un-packed (dim, width) weight so the speculative-decode
|
||||
# GDN path (which uses torch conv, not the AMX kernel) can use it.
|
||||
# GDN path (which uses torch conv, not the C++ kernel) can use it.
|
||||
layer._cpu_unpacked_conv_weight = unpacked
|
||||
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user