diff --git a/csrc/cpu/cpu_types_arm.hpp b/csrc/cpu/cpu_types_arm.hpp index 294dee90bd..46999e3b73 100644 --- a/csrc/cpu/cpu_types_arm.hpp +++ b/csrc/cpu/cpu_types_arm.hpp @@ -943,10 +943,14 @@ inline FP16Vec16::FP16Vec16(const FP32Vec16& v) { }; inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) { - fmadd(acc.reg.val[0], a.reg.val[0], b.reg.val[0]); - fmadd(acc.reg.val[1], a.reg.val[1], b.reg.val[1]); - fmadd(acc.reg.val[2], a.reg.val[2], b.reg.val[2]); - fmadd(acc.reg.val[3], a.reg.val[3], b.reg.val[3]); + // NOTE: at::vec::fmadd(a, b, c) returns a * b + c and does not mutate its + // arguments; we must capture the result back into acc, otherwise the FMA + // is a no-op. On x86 fma is implemented as `acc = acc + a * b` which does + // the assignment; the ARM version previously dropped the return value. + acc.reg.val[0] = fmadd(a.reg.val[0], b.reg.val[0], acc.reg.val[0]); + acc.reg.val[1] = fmadd(a.reg.val[1], b.reg.val[1], acc.reg.val[1]); + acc.reg.val[2] = fmadd(a.reg.val[2], b.reg.val[2], acc.reg.val[2]); + acc.reg.val[3] = fmadd(a.reg.val[3], b.reg.val[3], acc.reg.val[3]); }; inline BF16Vec8::BF16Vec8(const FP32Vec8& v) { @@ -967,10 +971,11 @@ inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) { std::tie(b0_low, b0_high) = convert_bfloat16_float(b.reg.val[0]); std::tie(b1_low, b1_high) = convert_bfloat16_float(b.reg.val[1]); - fmadd(acc.reg.val[0], a0_low, b0_low); - fmadd(acc.reg.val[1], a0_high, b0_high); - fmadd(acc.reg.val[2], a1_low, b1_low); - fmadd(acc.reg.val[3], a1_high, b1_high); + // Same fmadd semantic as the FP32 overload above: capture the result. + acc.reg.val[0] = fmadd(a0_low, b0_low, acc.reg.val[0]); + acc.reg.val[1] = fmadd(a0_high, b0_high, acc.reg.val[1]); + acc.reg.val[2] = fmadd(a1_low, b1_low, acc.reg.val[2]); + acc.reg.val[3] = fmadd(a1_high, b1_high, acc.reg.val[3]); }; template <> diff --git a/csrc/cpu/mla_decode.cpp b/csrc/cpu/mla_decode.cpp index 702912a5bc..bb41d57104 100644 --- a/csrc/cpu/mla_decode.cpp +++ b/csrc/cpu/mla_decode.cpp @@ -57,6 +57,12 @@ void mla_decode_block_head( std::fill(max_val, max_val + HEAD_UNROLL, -FLT_MAX); f32_vec_type acc_vec[BLOCK_SIZE][HEAD_UNROLL]; + // Explicitly zero-initialize the accumulator. On x86 the default + // ctor of FP32Vec16 already sets zero, but on ARM (and any backend + // whose Vectorized default ctor is trivial) the array holds + // undefined values and the fma below accumulates garbage. + for (int b = 0; b < BLOCK_SIZE; ++b) + for (int u = 0; u < HEAD_UNROLL; ++u) acc_vec[b][u] = f32_vec_type{0.0f}; for (int i = 0; i < HEAD_DIM; i += QK_NUM_ELEM) { // load to registers qk_vec_type q_vec[HEAD_UNROLL]; @@ -96,6 +102,9 @@ void mla_decode_block_head( } f32_vec_type this_out[V_HEAD_DIM / V_NUM_ELEM][HEAD_UNROLL]; + // Same zero-initialization concern as acc_vec above. + for (int i = 0; i < V_HEAD_DIM / V_NUM_ELEM; ++i) + for (int u = 0; u < HEAD_UNROLL; ++u) this_out[i][u] = f32_vec_type{0.0f}; for (int block_offset = 0; block_offset < num_tokens; ++block_offset) { // load to registers diff --git a/tests/v1/attention/test_cpu_mla_backend.py b/tests/v1/attention/test_cpu_mla_backend.py new file mode 100644 index 0000000000..18576ef076 --- /dev/null +++ b/tests/v1/attention/test_cpu_mla_backend.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Smoke tests for the CPU MLA backend. + +Two levels of coverage are provided: + +* :func:`test_kv_cache_cpu_write` checks that the kv-cache write logic + inlined in the CPU MLA backend writes the latent KV cache in the same + layout the CPU decode kernel expects. This runs without downloading + any model weights and is safe for CI. +* :func:`test_cpu_mla_backend_smoke` exercises the full end-to-end path + (LLM engine, model construction, prefill + decode) against a shrunk + DeepSeek-V2-Lite with `hf_overrides`. It uses `dummy` weights so the + outputs are not meaningful, only shape / plumbing correctness is + asserted. Marked `cpu_model` so it is skipped on non-CPU CI. +""" + +import pytest +import torch + +from vllm.platforms import current_platform + + +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") +def test_kv_cache_cpu_write() -> None: + kv_lora_rank = 512 + pe_dim = 64 + block_size = 16 + num_blocks = 4 + num_tokens = 8 + dtype = torch.float32 + + kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype) + k_pe = torch.randn(num_tokens, pe_dim, dtype=dtype) + kv_cache = torch.zeros(num_blocks, block_size, kv_lora_rank + pe_dim, dtype=dtype) + slot_mapping = torch.arange(num_tokens, dtype=torch.long) + # Mark the last slot as padding to make sure we honour negative slots. + slot_mapping[-1] = -1 + + flat = kv_cache.view(-1, kv_lora_rank + pe_dim) + slots = slot_mapping.to(torch.long) + valid_mask = slots >= 0 + if not valid_mask.all().item(): + slots = slots[valid_mask] + kv_c = kv_c[valid_mask] + k_pe = k_pe[valid_mask] + target_dtype = flat.dtype + flat[slots, :kv_lora_rank] = kv_c.to(target_dtype) + flat[slots, kv_lora_rank:] = k_pe.to(target_dtype) + + flat = kv_cache.view(-1, kv_lora_rank + pe_dim) + for i in range(num_tokens - 1): + assert torch.equal(flat[i, :kv_lora_rank], kv_c[i]) + assert torch.equal(flat[i, kv_lora_rank:], k_pe[i]) + # The padding slot must remain untouched. + assert torch.all(flat[num_tokens - 1] == 0) + + +@pytest.mark.cpu_model +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") +def test_cpu_mla_backend_smoke() -> None: + # Shrink the architecture aggressively so the test is CI-friendly. + from vllm import LLM, SamplingParams + + hf_overrides = { + "num_hidden_layers": 2, + "n_routed_experts": 4, + "first_k_dense_replace": 0, + "num_experts_per_tok": 2, + } + llm = LLM( + model="deepseek-ai/DeepSeek-V2-Lite", + trust_remote_code=True, + load_format="dummy", + enforce_eager=True, + max_model_len=128, + max_num_seqs=2, + block_size=16, + hf_overrides=hf_overrides, + ) + + sampling_params = SamplingParams(max_tokens=4, temperature=0.0) + outputs = llm.generate(["Hello", "MLA on CPU"], sampling_params) + assert len(outputs) == 2 + for output in outputs: + # `dummy` weights do not produce meaningful text, but the number + # of generated tokens must match what we asked for. + assert len(output.outputs[0].token_ids) == 4 diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 837038f7ae..139cfd6701 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -78,12 +78,19 @@ class CpuPlatform(Platform): attn_selector_config: "AttentionSelectorConfig", num_heads: int | None = None, ) -> str: - if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN: - logger.info("Cannot use %s backend on CPU.", selected_backend) - if attn_selector_config.use_mla: - raise NotImplementedError("MLA is not supported on CPU.") if attn_selector_config.use_sparse: raise NotImplementedError("Sparse Attention is not supported on CPU.") + if attn_selector_config.use_mla: + # Reference MLA implementation on CPU. Performance is not the + # goal here; the backend simply wires the CPU decode kernel + # (`mla_decode_kvcache`) and an SDPA-based prefill together with + # the shared MLA scaffolding so that DeepSeek-style models can + # execute on CPU. + if selected_backend and selected_backend != AttentionBackendEnum.CPU_MLA: + logger.info("Cannot use %s backend on CPU.", selected_backend) + return AttentionBackendEnum.CPU_MLA.get_path() + if selected_backend and selected_backend != AttentionBackendEnum.CPU_ATTN: + logger.info("Cannot use %s backend on CPU.", selected_backend) return AttentionBackendEnum.CPU_ATTN.get_path() @classmethod @@ -116,10 +123,25 @@ class CpuPlatform(Platform): cache_config = vllm_config.cache_config - if not cache_config.user_specified_block_size: + # The CPU MLA decode kernel only compiles with block_size=16 today + # (see csrc/cpu/mla_decode.cpp). If the model uses MLA we override + # the default block size regardless of user preference to avoid a + # runtime kernel dispatch failure. + cpu_mla_enabled = model_config is not None and getattr( + model_config, "use_mla", False + ) + if cpu_mla_enabled: + if cache_config.user_specified_block_size and cache_config.block_size != 16: + logger.warning( + "CPU MLA backend requires block_size=16, overriding " + "user-specified block_size=%s.", + cache_config.block_size, + ) + cache_config.block_size = 16 + elif not cache_config.user_specified_block_size: cache_config.block_size = 128 - if cache_config.block_size % 32 != 0: + if not cpu_mla_enabled and cache_config.block_size % 32 != 0: logger.warning( "CPU backend prefers block_size is multiples of 32, " "otherwise the performance is not optimized." @@ -321,6 +343,7 @@ class CpuPlatform(Platform): "prefill and prefix caching to be disabled." ) vllm_config.scheduler_config.enable_chunked_prefill = False + vllm_config.cache_config.enable_prefix_caching = False vllm_config.scheduler_config.max_num_batched_tokens = max( vllm_config.model_config.max_model_len, vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS, @@ -473,6 +496,13 @@ class CpuPlatform(Platform): from vllm._custom_ops import cpu_attn_reshape_and_cache from vllm.v1.attention.backends.cpu_attn import _get_attn_isa + # MLA uses a single latent cache of shape [N, block_size, head_size], + # so the classic key/value split does not apply. The MLA backend + # writes the cache itself via `concat_and_cache_mla` inside + # `do_kv_cache_update`, so there is nothing to pack here. + if kv_cache.dim() == 3: + return + num_blocks, num_kv_heads, block_size, fused_head_size = kv_cache.shape head_size = fused_head_size // 2 diff --git a/vllm/v1/attention/backends/mla/cpu_mla.py b/vllm/v1/attention/backends/mla/cpu_mla.py new file mode 100644 index 0000000000..91d3cb3339 --- /dev/null +++ b/vllm/v1/attention/backends/mla/cpu_mla.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU MLA backend. + +This is a *reference-quality* MLA backend for the CPU platform. It is +intended to make DeepSeek-V2/V3 style models runnable on CPU (for +functional verification, tiny-model smoke tests and CI), not to be +performant. The prefill path is a plain PyTorch SDPA and the decode +path forwards to the existing CPU decode kernel +`torch.ops._C.mla_decode_kvcache` (see csrc/cpu/mla_decode.cpp). + +Key design points: + +* We inherit the shared MLA scaffolding (`MLACommonBackend`, + `MLACommonImpl`, `MLACommonMetadata`) so that the same + ``forward_impl`` in ``MLAAttention`` orchestrates weight-absorbed + decode (MQA) and non-absorbed prefill (MHA) for us. +* The parent ``MLACommonImpl.__init__`` tries to pick a GPU prefill + kernel (flash_attn / flashinfer / cudnn / trtllm) and raises when + none of them are available. On CPU none of them apply, so we + bypass that logic and set the attributes ourselves. +* Chunked prefill and prefix caching are disabled by the CPU platform + when ``use_mla`` is set. +* The CPU decode kernel only supports ``head_dim=576``, + ``v_head_dim=512`` and ``block_size=16`` today; the platform layer + forces ``block_size=16`` for MLA models. +""" + +from typing import ClassVar + +import torch + +from vllm import _custom_ops as ops +from vllm.config.cache import CacheDType +from vllm.logger import init_logger +from vllm.model_executor.layers.attention.mla_attention import ( + MLACommonBackend, + MLACommonImpl, + MLACommonMetadata, +) +from vllm.v1.attention.backend import ( + AttentionLayer, + AttentionType, + MultipleOf, +) + +logger = init_logger(__name__) + + +class CPUMLABackend(MLACommonBackend): + """Attention backend descriptor for CPU MLA.""" + + supported_dtypes: ClassVar[list[torch.dtype]] = [ + torch.float16, + torch.bfloat16, + torch.float32, + ] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto"] + + @classmethod + def get_supported_head_sizes(cls) -> list[int]: + # DeepSeek-V2/V3 latent cache size = kv_lora_rank(512)+rope(64)=576. + # The current CPU decode kernel only compiles for this size. + return [576] + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + return [16] + + @classmethod + def supports_block_size(cls, block_size: int | None) -> bool: + return block_size is None or block_size == 16 + + @staticmethod + def get_name() -> str: + return "CPU_MLA" + + @staticmethod + def get_impl_cls() -> type["CPUMLAImpl"]: + return CPUMLAImpl + + +class CPUMLAImpl(MLACommonImpl[MLACommonMetadata]): + """CPU implementation of MLA attention. + + See module docstring for the overall design. This class only + overrides the parts of the parent that would otherwise pull in + CUDA-only dependencies. + """ + + # Decode always returns the softmax lse if the parent needs it, but + # CPU decode currently only produces the output tensor. Report False + # so DCP / cross-batch reductions are not requested from us. + can_return_lse_for_decode: bool = False + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: list[float] | None, + sliding_window: int | None, + kv_cache_dtype: str, + logits_soft_cap: float | None, + attn_type: str, + kv_sharing_target_layer_name: str | None, + # MLA-specific arguments (forwarded from `MLAAttention`). + **mla_args, + ) -> None: + # NOTE: We deliberately skip ``MLACommonImpl.__init__`` because it + # tries to pick a GPU prefill kernel and raises on CPU. Instead we + # replicate the attribute setup here and then wire our own hooks. + if kv_sharing_target_layer_name is not None: + raise NotImplementedError("KV sharing is not supported for MLA") + if attn_type != AttentionType.DECODER: + raise NotImplementedError("CPU MLA only supports decoder self-attention.") + unsupported = [alibi_slopes, sliding_window, logits_soft_cap] + if any(unsupported): + raise NotImplementedError( + "CPU MLA does not support alibi_slopes / sliding_window / " + "logits_soft_cap." + ) + + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + self.kv_cache_dtype = kv_cache_dtype + + self.q_lora_rank = mla_args["q_lora_rank"] + self.kv_lora_rank = mla_args["kv_lora_rank"] + self.qk_nope_head_dim = mla_args["qk_nope_head_dim"] + self.qk_rope_head_dim = mla_args["qk_rope_head_dim"] + self.qk_head_dim = mla_args["qk_head_dim"] + self.v_head_dim = mla_args["v_head_dim"] + self.kv_b_proj = mla_args["kv_b_proj"] + self.indexer = mla_args.get("indexer") + self.q_pad_num_heads = mla_args.get("q_pad_num_heads") + self.supports_quant_query_input = False + + # Never take the flashinfer specialisation of ``_concat_k_nope_k_pe``. + self._use_flashinfer_concat_mla_k = False + + # We do not need V padding since our SDPA prefill takes k / v with + # matching head dims. + self._pad_v = False + + # Distributed / context-parallel bookkeeping expected by the parent. + self.dcp_world_size = 1 + self.cp_kv_cache_interleave_size = 1 + + # ------------------------------------------------------------------ + # Decode (MQA) + # ------------------------------------------------------------------ + def forward_mqa( + self, + q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: MLACommonMetadata, + layer: AttentionLayer, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + assert attn_metadata.decode is not None + # The parent hands us either a tuple (q_nope, q_pe) or a single + # tensor already concatenated. CPU decode kernel wants a single + # (bs, num_heads, head_dim=kv_lora_rank+rope) contiguous tensor. + if isinstance(q, tuple): + q = torch.cat(q, dim=-1) + q = q.contiguous() + + bs, num_heads, _ = q.shape + out = torch.zeros( + bs, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device + ) + + # The CPU kernel is templated on int32 block_tables / seq_lens. + block_table = attn_metadata.decode.block_table.to(torch.int32) + seq_lens = attn_metadata.decode.seq_lens.to(torch.int32) + + ops.mla_decode_kvcache_cpu( + out, + q, + kv_c_and_k_pe_cache, + self.scale, + block_table, + seq_lens, + ) + # The parent only consumes the LSE when DCP is active, which we + # disable, so returning None here is safe. + return out, None + + # ------------------------------------------------------------------ + # KV cache write + # ------------------------------------------------------------------ + def do_kv_cache_update( + self, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + kv_cache: torch.Tensor, + slot_mapping: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor, + ) -> None: + if kv_cache.numel() == 0: + return + k_pe_sq = k_pe.squeeze(1) + kv_lora_rank = kv_c_normed.shape[-1] + pe_dim = k_pe_sq.shape[-1] + flat = kv_cache.view(-1, kv_lora_rank + pe_dim) + slots = slot_mapping.flatten().to(torch.long) + # Only write valid slots. Padding slots are set to -1 by the + # scheduler and must be skipped. + valid_mask = slots >= 0 + if not valid_mask.all().item(): + slots = slots[valid_mask] + kv_c_normed = kv_c_normed[valid_mask] + k_pe_sq = k_pe_sq[valid_mask] + target_dtype = flat.dtype + flat[slots, :kv_lora_rank] = kv_c_normed.to(target_dtype) + flat[slots, kv_lora_rank:] = k_pe_sq.view(-1, pe_dim).to(target_dtype) diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index 5ed38ecedb..646227cc4f 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -122,6 +122,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): "RocmAiterUnifiedAttentionBackend" ) CPU_ATTN = "vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend" + CPU_MLA = "vllm.v1.attention.backends.mla.cpu_mla.CPUMLABackend" TURBOQUANT = "vllm.v1.attention.backends.turboquant_attn.TurboQuantAttentionBackend" # Placeholder for third-party/custom backends - must be registered before use # set to None to avoid alias with other backend, whose value is an empty string