[CPU][MLA] Fix prefill backend selection so MLA runs end-to-end on CPU (#51471)

Signed-off-by: baoloongmao <baoloongmao@tencent.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
maobaolong
2026-08-28 12:36:55 +08:00
committed by GitHub
co-authored by Codex
parent e6bfe03ad7
commit 6f7df92a8e
9 changed files with 442 additions and 69 deletions
+25
View File
@@ -174,6 +174,31 @@ def mla_decode_kvcache_cpu(
torch.ops._C.mla_decode_kvcache(out, query, kv_cache, scale, block_tables, seq_lens)
def gather_mla_context_cache_cpu(
*,
src_cache: torch.Tensor,
dst: torch.Tensor,
block_table: torch.Tensor,
starts: torch.Tensor,
cu_seq_lens: torch.Tensor,
) -> None:
page_size = src_cache.shape[1]
flat_cache = src_cache.view(-1, src_cache.shape[-1])
seq_lens = cu_seq_lens[1:] - cu_seq_lens[:-1]
out_start = 0
for req_idx, (start, seq_len) in enumerate(zip(starts.tolist(), seq_lens.tolist())):
if seq_len <= 0:
continue
positions = torch.arange(start, start + seq_len, device=block_table.device)
block_ids = block_table[req_idx, positions // page_size].to(torch.long)
offsets = positions % page_size
slots = block_ids * page_size + offsets
out_end = out_start + seq_len
dst[out_start:out_end].copy_(flat_cache[slots])
out_start = out_end
# merge attn states ops
def merge_attn_states(
output: torch.Tensor,
@@ -2785,6 +2785,18 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
batch_size=chunk.num_requests,
seq_starts=chunk.starts,
)
elif current_platform.is_cpu():
assert not is_quantized_kv_cache(self.kv_cache_dtype), (
"CPU MLA context gather fallback only supports "
f"non-quantized KV cache, got {self.kv_cache_dtype}"
)
ops.gather_mla_context_cache_cpu(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=block_table,
starts=chunk.starts,
cu_seq_lens=chunk.cu_seq_lens,
)
elif not use_fp8_prefill:
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inert MLA prefill backend for the CPU platform.
CPU's MLA attention impl (``CPUMLAImpl``) fully overrides ``forward_mha`` with a
kernel that attends directly against the paged latent KV cache (covering both
fresh prefill and cached-prefix continuation in one pass), so it never calls
into a pluggable ``MLAPrefillBackend``'s ``run_prefill_new_tokens``/
``run_prefill_context_chunk``. This class exists only to satisfy
``MLACommonMetadataBuilder``'s structural requirement that
``MLAAttention.prefill_backend`` be a real, cloneable object.
"""
from typing import TYPE_CHECKING
import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonPrefillMetadata,
)
class CPUNativeMLAPrefillBackend(MLAPrefillBackend):
"""Placeholder MLA prefill backend for CPU; never actually invoked."""
@staticmethod
def get_name() -> str:
return "CPU_NATIVE"
def run_prefill_new_tokens(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
return_softmax_lse: bool,
out: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
raise AssertionError(
"CPUNativeMLAPrefillBackend.run_prefill_new_tokens is unreachable: "
"CPUMLAImpl.forward_mha fully overrides the dense-MHA prefill path "
"and never calls the pluggable prefill backend."
)
def run_prefill_context_chunk(
self,
chunk: "MLACommonPrefillMetadata.ContextChunk",
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
raise AssertionError(
"CPUNativeMLAPrefillBackend.run_prefill_context_chunk is "
"unreachable: CPUMLAImpl.forward_mha fully overrides the "
"dense-MHA prefill path and never calls the pluggable prefill "
"backend."
)
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU MLA prefill backend implemented with dense per-request attention.
This backend is selected on CPU devices, where flash-attn is unavailable. It
computes MLA prefill attention by slicing the ragged (varlen) q/k/v tensors
per request and evaluating dense attention in PyTorch.
"""
from typing import TYPE_CHECKING
import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonPrefillMetadata,
)
class CPUSDPAMLAPrefillBackend(MLAPrefillBackend):
"""MLA prefill backend for CPU using dense PyTorch attention.
flash-attn is not installable on CPU, so this backend provides an
equivalent prefill implementation using per-request dense attention.
"""
@staticmethod
def get_name() -> str:
return "CPU_SDPA_MLA"
@classmethod
def is_available(cls) -> bool:
# SDPA is always available on CPU.
return True
def supports_out(self) -> bool:
return True
def _ragged_sdpa(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_cu_seq_lens: torch.Tensor,
kv_cu_seq_lens: torch.Tensor,
causal: bool,
return_softmax_lse: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
# q/k/v: [num_tokens, num_heads, head_dim] (ragged, concatenated).
# q_cu_seq_lens/kv_cu_seq_lens: [num_reqs + 1] row offsets for each
# request's query and key/value tokens.
num_reqs = q_cu_seq_lens.numel() - 1
assert num_reqs == kv_cu_seq_lens.numel() - 1
out_chunks: list[torch.Tensor] = []
lse_chunks: list[torch.Tensor] = []
for r in range(num_reqs):
q_start = int(q_cu_seq_lens[r])
q_end = int(q_cu_seq_lens[r + 1])
kv_start = int(kv_cu_seq_lens[r])
kv_end = int(kv_cu_seq_lens[r + 1])
q_r = q[q_start:q_end].transpose(0, 1).float() # [NH, Sq, Hq]
k_r = k[kv_start:kv_end].transpose(0, 1).float() # [NH, Sk, Hq]
v_r = v[kv_start:kv_end].transpose(0, 1).float() # [NH, Sk, Hv]
attn_scores = torch.matmul(q_r, k_r.transpose(-2, -1)) * self.scale
if causal:
causal_mask = torch.ones(
(q_end - q_start, kv_end - kv_start),
dtype=torch.bool,
device=attn_scores.device,
).triu(1)
attn_scores.masked_fill_(causal_mask.unsqueeze(0), float("-inf"))
attn_probs = torch.softmax(attn_scores, dim=-1, dtype=torch.float32)
out_r = torch.matmul(attn_probs, v_r).transpose(0, 1).to(v.dtype)
out_chunks.append(out_r) # [Sq, NH, Hv]
if return_softmax_lse:
lse_chunks.append(torch.logsumexp(attn_scores, dim=-1))
out = torch.cat(out_chunks, dim=0)
if not return_softmax_lse:
return out
lse = torch.cat(lse_chunks, dim=1)
return out, lse
def run_prefill_new_tokens(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
return_softmax_lse: bool,
out: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
assert output_scale is None, (
"CPU MLA prefill backend does not support fused FP8 output"
)
out_t = self._ragged_sdpa(
q,
k,
v,
self._prefill_metadata.query_start_loc,
self._prefill_metadata.query_start_loc,
causal=True,
return_softmax_lse=return_softmax_lse,
)
if out is not None and not return_softmax_lse:
assert isinstance(out_t, torch.Tensor)
out.copy_(out_t)
return out
return out_t
def run_prefill_context_chunk(
self,
chunk: "MLACommonPrefillMetadata.ContextChunk",
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert out is None, "CPU MLA context prefill does not support in-place output"
result = self._ragged_sdpa(
q,
k,
v,
chunk.query_start_loc,
chunk.cu_seq_lens,
causal=False,
return_softmax_lse=True,
)
assert isinstance(result, tuple)
out, lse = result
return out, lse
@@ -52,9 +52,7 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
"vllm.v1.attention.backends.mla.prefill.aiter_flash_attn."
"AiterFlashAttnPrefillBackend"
)
CPU_NATIVE = (
"vllm.v1.attention.backends.mla.prefill.cpu_native.CPUNativeMLAPrefillBackend"
)
CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend"
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string
CUSTOM = None
@@ -107,10 +107,12 @@ def get_mla_prefill_backend(
"""
from vllm.platforms import current_platform
if current_platform.is_cpu():
logger.info_once("Using CPU SDPA MLA prefill backend.")
return MLAPrefillBackendEnum.CPU.get_class()
device_capability = current_platform.get_device_capability()
if device_capability is None:
if current_platform.is_cpu():
return MLAPrefillBackendEnum.CPU_NATIVE.get_class()
logger.info_once(
"Device capability not available, using FlashAttention MLA prefill backend."
)
@@ -6,6 +6,68 @@ import torch
from vllm.platforms import current_platform
def _merge_attn_states_torch(
output: torch.Tensor,
prefix_output: torch.Tensor,
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output_lse: torch.Tensor | None = None,
prefill_tokens_with_context: int | None = None,
output_scale: torch.Tensor | None = None,
) -> None:
assert output_scale is None, "CPU fallback does not support FP8 output"
num_tokens = output.shape[0]
if prefill_tokens_with_context is None:
prefill_tokens_with_context = num_tokens
prefix_lse = torch.where(
torch.isinf(prefix_lse) & (prefix_lse > 0),
torch.full_like(prefix_lse, float("-inf")),
prefix_lse,
)
suffix_lse = torch.where(
torch.isinf(suffix_lse) & (suffix_lse > 0),
torch.full_like(suffix_lse, float("-inf")),
suffix_lse,
)
if prefill_tokens_with_context < num_tokens:
output[prefill_tokens_with_context:].copy_(
suffix_output[prefill_tokens_with_context:]
)
if output_lse is not None:
output_lse[:, prefill_tokens_with_context:].copy_(
suffix_lse[:, prefill_tokens_with_context:]
)
if prefill_tokens_with_context == 0:
return
prefix_lse = prefix_lse[:, :prefill_tokens_with_context].float()
suffix_lse = suffix_lse[:, :prefill_tokens_with_context].float()
max_lse = torch.maximum(prefix_lse, suffix_lse)
prefix_se = torch.exp(prefix_lse - max_lse)
suffix_se = torch.exp(suffix_lse - max_lse)
out_se = prefix_se + suffix_se
both_empty = max_lse == float("-inf")
prefix_scale = (prefix_se / out_se).transpose(0, 1).unsqueeze(-1)
suffix_scale = (suffix_se / out_se).transpose(0, 1).unsqueeze(-1)
merged = (
prefix_output[:prefill_tokens_with_context].float() * prefix_scale
+ suffix_output[:prefill_tokens_with_context].float() * suffix_scale
)
merged.masked_fill_(both_empty.transpose(0, 1).unsqueeze(-1), 0.0)
output[:prefill_tokens_with_context].copy_(merged.to(output.dtype))
if output_lse is not None:
merged_lse = torch.log(out_se) + max_lse
merged_lse.masked_fill_(both_empty, float("-inf"))
output_lse[:, :prefill_tokens_with_context].copy_(merged_lse)
def merge_attn_states(
output: torch.Tensor,
prefix_output: torch.Tensor,
@@ -94,6 +156,17 @@ def merge_attn_states(
prefill_tokens_with_context,
output_scale,
)
elif current_platform.is_cpu():
return _merge_attn_states_torch(
output,
prefix_output,
prefix_lse,
suffix_output,
suffix_lse,
output_lse,
prefill_tokens_with_context,
output_scale,
)
else:
from vllm.v1.attention.ops.triton_merge_attn_states import (
merge_attn_states,