[CPU] Add MLA backend so DeepSeek-V2/V3 can run on CPU (#49453)

Signed-off-by: baoloongmao <baoloongmao@tencent.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
maobaolong
2026-08-06 16:51:41 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent ad5280255b
commit 13726c80fe
6 changed files with 367 additions and 14 deletions
+13 -8
View File
@@ -943,10 +943,14 @@ inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
};
inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
fmadd(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
fmadd(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
fmadd(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
fmadd(acc.reg.val[3], a.reg.val[3], b.reg.val[3]);
// NOTE: at::vec::fmadd(a, b, c) returns a * b + c and does not mutate its
// arguments; we must capture the result back into acc, otherwise the FMA
// is a no-op. On x86 fma is implemented as `acc = acc + a * b` which does
// the assignment; the ARM version previously dropped the return value.
acc.reg.val[0] = fmadd(a.reg.val[0], b.reg.val[0], acc.reg.val[0]);
acc.reg.val[1] = fmadd(a.reg.val[1], b.reg.val[1], acc.reg.val[1]);
acc.reg.val[2] = fmadd(a.reg.val[2], b.reg.val[2], acc.reg.val[2]);
acc.reg.val[3] = fmadd(a.reg.val[3], b.reg.val[3], acc.reg.val[3]);
};
inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
@@ -967,10 +971,11 @@ inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
std::tie(b0_low, b0_high) = convert_bfloat16_float(b.reg.val[0]);
std::tie(b1_low, b1_high) = convert_bfloat16_float(b.reg.val[1]);
fmadd(acc.reg.val[0], a0_low, b0_low);
fmadd(acc.reg.val[1], a0_high, b0_high);
fmadd(acc.reg.val[2], a1_low, b1_low);
fmadd(acc.reg.val[3], a1_high, b1_high);
// Same fmadd semantic as the FP32 overload above: capture the result.
acc.reg.val[0] = fmadd(a0_low, b0_low, acc.reg.val[0]);
acc.reg.val[1] = fmadd(a0_high, b0_high, acc.reg.val[1]);
acc.reg.val[2] = fmadd(a1_low, b1_low, acc.reg.val[2]);
acc.reg.val[3] = fmadd(a1_high, b1_high, acc.reg.val[3]);
};
template <>
+9
View File
@@ -57,6 +57,12 @@ void mla_decode_block_head(
std::fill(max_val, max_val + HEAD_UNROLL, -FLT_MAX);
f32_vec_type acc_vec[BLOCK_SIZE][HEAD_UNROLL];
// Explicitly zero-initialize the accumulator. On x86 the default
// ctor of FP32Vec16 already sets zero, but on ARM (and any backend
// whose Vectorized<float> default ctor is trivial) the array holds
// undefined values and the fma below accumulates garbage.
for (int b = 0; b < BLOCK_SIZE; ++b)
for (int u = 0; u < HEAD_UNROLL; ++u) acc_vec[b][u] = f32_vec_type{0.0f};
for (int i = 0; i < HEAD_DIM; i += QK_NUM_ELEM) {
// load to registers
qk_vec_type q_vec[HEAD_UNROLL];
@@ -96,6 +102,9 @@ void mla_decode_block_head(
}
f32_vec_type this_out[V_HEAD_DIM / V_NUM_ELEM][HEAD_UNROLL];
// Same zero-initialization concern as acc_vec above.
for (int i = 0; i < V_HEAD_DIM / V_NUM_ELEM; ++i)
for (int u = 0; u < HEAD_UNROLL; ++u) this_out[i][u] = f32_vec_type{0.0f};
for (int block_offset = 0; block_offset < num_tokens; ++block_offset) {
// load to registers
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Smoke tests for the CPU MLA backend.
Two levels of coverage are provided:
* :func:`test_kv_cache_cpu_write` checks that the kv-cache write logic
inlined in the CPU MLA backend writes the latent KV cache in the same
layout the CPU decode kernel expects. This runs without downloading
any model weights and is safe for CI.
* :func:`test_cpu_mla_backend_smoke` exercises the full end-to-end path
(LLM engine, model construction, prefill + decode) against a shrunk
DeepSeek-V2-Lite with `hf_overrides`. It uses `dummy` weights so the
outputs are not meaningful, only shape / plumbing correctness is
asserted. Marked `cpu_model` so it is skipped on non-CPU CI.
"""
import pytest
import torch
from vllm.platforms import current_platform
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_kv_cache_cpu_write() -> None:
kv_lora_rank = 512
pe_dim = 64
block_size = 16
num_blocks = 4
num_tokens = 8
dtype = torch.float32
kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype)
k_pe = torch.randn(num_tokens, pe_dim, dtype=dtype)
kv_cache = torch.zeros(num_blocks, block_size, kv_lora_rank + pe_dim, dtype=dtype)
slot_mapping = torch.arange(num_tokens, dtype=torch.long)
# Mark the last slot as padding to make sure we honour negative slots.
slot_mapping[-1] = -1
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
slots = slot_mapping.to(torch.long)
valid_mask = slots >= 0
if not valid_mask.all().item():
slots = slots[valid_mask]
kv_c = kv_c[valid_mask]
k_pe = k_pe[valid_mask]
target_dtype = flat.dtype
flat[slots, :kv_lora_rank] = kv_c.to(target_dtype)
flat[slots, kv_lora_rank:] = k_pe.to(target_dtype)
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
for i in range(num_tokens - 1):
assert torch.equal(flat[i, :kv_lora_rank], kv_c[i])
assert torch.equal(flat[i, kv_lora_rank:], k_pe[i])
# The padding slot must remain untouched.
assert torch.all(flat[num_tokens - 1] == 0)
@pytest.mark.cpu_model
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_cpu_mla_backend_smoke() -> None:
# Shrink the architecture aggressively so the test is CI-friendly.
from vllm import LLM, SamplingParams
hf_overrides = {
"num_hidden_layers": 2,
"n_routed_experts": 4,
"first_k_dense_replace": 0,
"num_experts_per_tok": 2,
}
llm = LLM(
model="deepseek-ai/DeepSeek-V2-Lite",
trust_remote_code=True,
load_format="dummy",
enforce_eager=True,
max_model_len=128,
max_num_seqs=2,
block_size=16,
hf_overrides=hf_overrides,
)
sampling_params = SamplingParams(max_tokens=4, temperature=0.0)
outputs = llm.generate(["Hello", "MLA on CPU"], sampling_params)
assert len(outputs) == 2
for output in outputs:
# `dummy` weights do not produce meaningful text, but the number
# of generated tokens must match what we asked for.
assert len(output.outputs[0].token_ids) == 4
+36 -6
View File
@@ -78,12 +78,19 @@ class CpuPlatform(Platform):
attn_selector_config: "AttentionSelectorConfig",
num_heads: int | None = None,
) -> str:
if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN:
logger.info("Cannot use %s backend on CPU.", selected_backend)
if attn_selector_config.use_mla:
raise NotImplementedError("MLA is not supported on CPU.")
if attn_selector_config.use_sparse:
raise NotImplementedError("Sparse Attention is not supported on CPU.")
if attn_selector_config.use_mla:
# Reference MLA implementation on CPU. Performance is not the
# goal here; the backend simply wires the CPU decode kernel
# (`mla_decode_kvcache`) and an SDPA-based prefill together with
# the shared MLA scaffolding so that DeepSeek-style models can
# execute on CPU.
if selected_backend and selected_backend != AttentionBackendEnum.CPU_MLA:
logger.info("Cannot use %s backend on CPU.", selected_backend)
return AttentionBackendEnum.CPU_MLA.get_path()
if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN:
logger.info("Cannot use %s backend on CPU.", selected_backend)
return AttentionBackendEnum.CPU_ATTN.get_path()
@classmethod
@@ -116,10 +123,25 @@ class CpuPlatform(Platform):
cache_config = vllm_config.cache_config
if not cache_config.user_specified_block_size:
# The CPU MLA decode kernel only compiles with block_size=16 today
# (see csrc/cpu/mla_decode.cpp). If the model uses MLA we override
# the default block size regardless of user preference to avoid a
# runtime kernel dispatch failure.
cpu_mla_enabled = model_config is not None and getattr(
model_config, "use_mla", False
)
if cpu_mla_enabled:
if cache_config.user_specified_block_size and cache_config.block_size != 16:
logger.warning(
"CPU MLA backend requires block_size=16, overriding "
"user-specified block_size=%s.",
cache_config.block_size,
)
cache_config.block_size = 16
elif not cache_config.user_specified_block_size:
cache_config.block_size = 128
if cache_config.block_size % 32 != 0:
if not cpu_mla_enabled and cache_config.block_size % 32 != 0:
logger.warning(
"CPU backend prefers block_size is multiples of 32, "
"otherwise the performance is not optimized."
@@ -321,6 +343,7 @@ class CpuPlatform(Platform):
"prefill and prefix caching to be disabled."
)
vllm_config.scheduler_config.enable_chunked_prefill = False
vllm_config.cache_config.enable_prefix_caching = False
vllm_config.scheduler_config.max_num_batched_tokens = max(
vllm_config.model_config.max_model_len,
vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
@@ -473,6 +496,13 @@ class CpuPlatform(Platform):
from vllm._custom_ops import cpu_attn_reshape_and_cache
from vllm.v1.attention.backends.cpu_attn import _get_attn_isa
# MLA uses a single latent cache of shape [N, block_size, head_size],
# so the classic key/value split does not apply. The MLA backend
# writes the cache itself via `concat_and_cache_mla` inside
# `do_kv_cache_update`, so there is nothing to pack here.
if kv_cache.dim() == 3:
return
num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape
head_size = fused_head_size // 2
+220
View File
@@ -0,0 +1,220 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU MLA backend.
This is a *reference-quality* MLA backend for the CPU platform. It is
intended to make DeepSeek-V2/V3 style models runnable on CPU (for
functional verification, tiny-model smoke tests and CI), not to be
performant. The prefill path is a plain PyTorch SDPA and the decode
path forwards to the existing CPU decode kernel
`torch.ops._C.mla_decode_kvcache` (see csrc/cpu/mla_decode.cpp).
Key design points:
* We inherit the shared MLA scaffolding (`MLACommonBackend`,
`MLACommonImpl`, `MLACommonMetadata`) so that the same
``forward_impl`` in ``MLAAttention`` orchestrates weight-absorbed
decode (MQA) and non-absorbed prefill (MHA) for us.
* The parent ``MLACommonImpl.__init__`` tries to pick a GPU prefill
kernel (flash_attn / flashinfer / cudnn / trtllm) and raises when
none of them are available. On CPU none of them apply, so we
bypass that logic and set the attributes ourselves.
* Chunked prefill and prefix caching are disabled by the CPU platform
when ``use_mla`` is set.
* The CPU decode kernel only supports ``head_dim=576``,
``v_head_dim=512`` and ``block_size=16`` today; the platform layer
forces ``block_size=16`` for MLA models.
"""
from typing import ClassVar
import torch
from vllm import _custom_ops as ops
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonImpl,
MLACommonMetadata,
)
from vllm.v1.attention.backend import (
AttentionLayer,
AttentionType,
MultipleOf,
)
logger = init_logger(__name__)
class CPUMLABackend(MLACommonBackend):
"""Attention backend descriptor for CPU MLA."""
supported_dtypes: ClassVar[list[torch.dtype]] = [
torch.float16,
torch.bfloat16,
torch.float32,
]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto"]
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
# DeepSeek-V2/V3 latent cache size = kv_lora_rank(512)+rope(64)=576.
# The current CPU decode kernel only compiles for this size.
return [576]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [16]
@classmethod
def supports_block_size(cls, block_size: int | None) -> bool:
return block_size is None or block_size == 16
@staticmethod
def get_name() -> str:
return "CPU_MLA"
@staticmethod
def get_impl_cls() -> type["CPUMLAImpl"]:
return CPUMLAImpl
class CPUMLAImpl(MLACommonImpl[MLACommonMetadata]):
"""CPU implementation of MLA attention.
See module docstring for the overall design. This class only
overrides the parts of the parent that would otherwise pull in
CUDA-only dependencies.
"""
# Decode always returns the softmax lse if the parent needs it, but
# CPU decode currently only produces the output tensor. Report False
# so DCP / cross-batch reductions are not requested from us.
can_return_lse_for_decode: bool = False
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
# MLA-specific arguments (forwarded from `MLAAttention`).
**mla_args,
) -> None:
# NOTE: We deliberately skip ``MLACommonImpl.__init__`` because it
# tries to pick a GPU prefill kernel and raises on CPU. Instead we
# replicate the attribute setup here and then wire our own hooks.
if kv_sharing_target_layer_name is not None:
raise NotImplementedError("KV sharing is not supported for MLA")
if attn_type != AttentionType.DECODER:
raise NotImplementedError("CPU MLA only supports decoder self-attention.")
unsupported = [alibi_slopes, sliding_window, logits_soft_cap]
if any(unsupported):
raise NotImplementedError(
"CPU MLA does not support alibi_slopes / sliding_window / "
"logits_soft_cap."
)
self.num_heads = num_heads
self.head_size = head_size
self.scale = float(scale)
self.num_kv_heads = num_kv_heads
self.kv_cache_dtype = kv_cache_dtype
self.q_lora_rank = mla_args["q_lora_rank"]
self.kv_lora_rank = mla_args["kv_lora_rank"]
self.qk_nope_head_dim = mla_args["qk_nope_head_dim"]
self.qk_rope_head_dim = mla_args["qk_rope_head_dim"]
self.qk_head_dim = mla_args["qk_head_dim"]
self.v_head_dim = mla_args["v_head_dim"]
self.kv_b_proj = mla_args["kv_b_proj"]
self.indexer = mla_args.get("indexer")
self.q_pad_num_heads = mla_args.get("q_pad_num_heads")
self.supports_quant_query_input = False
# Never take the flashinfer specialisation of ``_concat_k_nope_k_pe``.
self._use_flashinfer_concat_mla_k = False
# We do not need V padding since our SDPA prefill takes k / v with
# matching head dims.
self._pad_v = False
# Distributed / context-parallel bookkeeping expected by the parent.
self.dcp_world_size = 1
self.cp_kv_cache_interleave_size = 1
# ------------------------------------------------------------------
# Decode (MQA)
# ------------------------------------------------------------------
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert attn_metadata.decode is not None
# The parent hands us either a tuple (q_nope, q_pe) or a single
# tensor already concatenated. CPU decode kernel wants a single
# (bs, num_heads, head_dim=kv_lora_rank+rope) contiguous tensor.
if isinstance(q, tuple):
q = torch.cat(q, dim=-1)
q = q.contiguous()
bs, num_heads, _ = q.shape
out = torch.zeros(
bs, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device
)
# The CPU kernel is templated on int32 block_tables / seq_lens.
block_table = attn_metadata.decode.block_table.to(torch.int32)
seq_lens = attn_metadata.decode.seq_lens.to(torch.int32)
ops.mla_decode_kvcache_cpu(
out,
q,
kv_c_and_k_pe_cache,
self.scale,
block_table,
seq_lens,
)
# The parent only consumes the LSE when DCP is active, which we
# disable, so returning None here is safe.
return out, None
# ------------------------------------------------------------------
# KV cache write
# ------------------------------------------------------------------
def do_kv_cache_update(
self,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
kv_cache_dtype: str,
k_scale: torch.Tensor,
) -> None:
if kv_cache.numel() == 0:
return
k_pe_sq = k_pe.squeeze(1)
kv_lora_rank = kv_c_normed.shape[-1]
pe_dim = k_pe_sq.shape[-1]
flat = kv_cache.view(-1, kv_lora_rank + pe_dim)
slots = slot_mapping.flatten().to(torch.long)
# Only write valid slots. Padding slots are set to -1 by the
# scheduler and must be skipped.
valid_mask = slots >= 0
if not valid_mask.all().item():
slots = slots[valid_mask]
kv_c_normed = kv_c_normed[valid_mask]
k_pe_sq = k_pe_sq[valid_mask]
target_dtype = flat.dtype
flat[slots, :kv_lora_rank] = kv_c_normed.to(target_dtype)
flat[slots, kv_lora_rank:] = k_pe_sq.view(-1, pe_dim).to(target_dtype)
+1
View File
@@ -122,6 +122,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
"RocmAiterUnifiedAttentionBackend"
)
CPU_ATTN = "vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend"
CPU_MLA = "vllm.v1.attention.backends.mla.cpu_mla.CPUMLABackend"
TURBOQUANT = "vllm.v1.attention.backends.turboquant_attn.TurboQuantAttentionBackend"
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string