[CPU] Add AMX-only high-performance MLA backend for DeepSeek V2/V3/R1 (#52616)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-08-19 10:28:52 +08:00
committed by GitHub
parent f485081e8b
commit e4d61d0d22
20 changed files with 4156 additions and 6 deletions
@@ -540,6 +540,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
**extra_impl_args,
)
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
# AMX reads kv_b_proj's weight directly and never calls it live; the
# reference CPU MLA backend calls it but isn't perf-critical. Skip
# the packed-kernel dispatch either way.
kv_b_proj._cpu_skip_gemm_dispatch = True
self.use_direct_call = not current_platform.opaque_attention_op()
vllm_config = get_current_vllm_config()
@@ -908,6 +913,20 @@ class MLAAttention(nn.Module, AttentionLayerBase):
group_size=128,
transpose_bm=True,
)
elif self.is_amx_bmm_enabled:
# bmm_cpu computes out[n] = mat1[n] @ mat2[n]^T against
# AMXMLAImpl's own (N, L, P) packed W_UK -- same as prefill.
N, B, P = mqa_q_nope.shape
L = self.kv_lora_rank
mqa_ql_nope = mqa_q_nope.new_empty((N, B, L))
ops.bmm_cpu(
mqa_ql_nope,
mqa_q_nope,
self.impl._w_uk_packed, # type: ignore[attr-defined]
True,
None,
)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
else:
# Pads the head_dim if necessary (for the underlying kernel)
N, B, P = mqa_q_nope.shape
@@ -1026,6 +1045,18 @@ class MLAAttention(nn.Module, AttentionLayerBase):
return output_padded
def process_weights_after_loading(self, act_dtype: torch.dtype):
# Let per-backend impls do their own weight packing first (no-op
# unless overridden), mirroring Attention.process_weights_after_loading.
self.impl.process_weights_after_loading(act_dtype)
if self.is_amx_bmm_enabled:
# AMXMLAImpl already packed its own W_UK/W_UV above, for both
# prefill and decode. Release the now-unused raw weight.
self.kv_b_proj.weight = torch.nn.Parameter(
torch.empty(0), requires_grad=False
)
return
# we currently do not have quantized bmm's which are needed for
# `W_UV` and `W_UK_T`, we just store fp16/bf16 copies and perform
# the bmm's in 16-bit, the extra memory overhead of this is fairly low
@@ -1192,6 +1223,16 @@ class MLAAttention(nn.Module, AttentionLayerBase):
x = rocm_aiter_ops.triton_fp8_bmm(
x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True, YQ=out
)
elif self.is_amx_bmm_enabled:
# bmm_cpu computes out[n] = mat1[n] @ mat2[n]^T against
# AMXMLAImpl's own (N, V, L) packed W_UV -- same as prefill.
ops.bmm_cpu(
out.transpose(0, 1),
x,
self.impl._w_uv_packed, # type: ignore[attr-defined]
True,
None,
)
else:
# Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V)
torch.bmm(x, self.W_UV, out=out.transpose(0, 1))
+6
View File
@@ -197,6 +197,12 @@ class UnquantizedLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if current_platform.is_cpu():
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): not
# perf-critical, so skip packing and use a plain fallback.
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
layer.cpu_linear = torch.nn.functional.linear
return
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
dispatch_cpu_unquantized_gemm(layer, remove_weight=True)