[CPU][MLA] Fix prefill backend selection so MLA runs end-to-end on CPU (#51471)
Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -174,6 +174,31 @@ def mla_decode_kvcache_cpu(
|
||||
torch.ops._C.mla_decode_kvcache(out, query, kv_cache, scale, block_tables, seq_lens)
|
||||
|
||||
|
||||
def gather_mla_context_cache_cpu(
|
||||
*,
|
||||
src_cache: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
starts: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
) -> None:
|
||||
page_size = src_cache.shape[1]
|
||||
flat_cache = src_cache.view(-1, src_cache.shape[-1])
|
||||
seq_lens = cu_seq_lens[1:] - cu_seq_lens[:-1]
|
||||
|
||||
out_start = 0
|
||||
for req_idx, (start, seq_len) in enumerate(zip(starts.tolist(), seq_lens.tolist())):
|
||||
if seq_len <= 0:
|
||||
continue
|
||||
positions = torch.arange(start, start + seq_len, device=block_table.device)
|
||||
block_ids = block_table[req_idx, positions // page_size].to(torch.long)
|
||||
offsets = positions % page_size
|
||||
slots = block_ids * page_size + offsets
|
||||
out_end = out_start + seq_len
|
||||
dst[out_start:out_end].copy_(flat_cache[slots])
|
||||
out_start = out_end
|
||||
|
||||
|
||||
# merge attn states ops
|
||||
def merge_attn_states(
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -2785,6 +2785,18 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
assert not is_quantized_kv_cache(self.kv_cache_dtype), (
|
||||
"CPU MLA context gather fallback only supports "
|
||||
f"non-quantized KV cache, got {self.kv_cache_dtype}"
|
||||
)
|
||||
ops.gather_mla_context_cache_cpu(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
starts=chunk.starts,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
)
|
||||
elif not use_fp8_prefill:
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inert MLA prefill backend for the CPU platform.
|
||||
|
||||
CPU's MLA attention impl (``CPUMLAImpl``) fully overrides ``forward_mha`` with a
|
||||
kernel that attends directly against the paged latent KV cache (covering both
|
||||
fresh prefill and cached-prefix continuation in one pass), so it never calls
|
||||
into a pluggable ``MLAPrefillBackend``'s ``run_prefill_new_tokens``/
|
||||
``run_prefill_context_chunk``. This class exists only to satisfy
|
||||
``MLACommonMetadataBuilder``'s structural requirement that
|
||||
``MLAAttention.prefill_backend`` be a real, cloneable object.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonPrefillMetadata,
|
||||
)
|
||||
|
||||
|
||||
class CPUNativeMLAPrefillBackend(MLAPrefillBackend):
|
||||
"""Placeholder MLA prefill backend for CPU; never actually invoked."""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_NATIVE"
|
||||
|
||||
def run_prefill_new_tokens(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
return_softmax_lse: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
raise AssertionError(
|
||||
"CPUNativeMLAPrefillBackend.run_prefill_new_tokens is unreachable: "
|
||||
"CPUMLAImpl.forward_mha fully overrides the dense-MHA prefill path "
|
||||
"and never calls the pluggable prefill backend."
|
||||
)
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
self,
|
||||
chunk: "MLACommonPrefillMetadata.ContextChunk",
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
raise AssertionError(
|
||||
"CPUNativeMLAPrefillBackend.run_prefill_context_chunk is "
|
||||
"unreachable: CPUMLAImpl.forward_mha fully overrides the "
|
||||
"dense-MHA prefill path and never calls the pluggable prefill "
|
||||
"backend."
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU MLA prefill backend implemented with dense per-request attention.
|
||||
|
||||
This backend is selected on CPU devices, where flash-attn is unavailable. It
|
||||
computes MLA prefill attention by slicing the ragged (varlen) q/k/v tensors
|
||||
per request and evaluating dense attention in PyTorch.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonPrefillMetadata,
|
||||
)
|
||||
|
||||
|
||||
class CPUSDPAMLAPrefillBackend(MLAPrefillBackend):
|
||||
"""MLA prefill backend for CPU using dense PyTorch attention.
|
||||
|
||||
flash-attn is not installable on CPU, so this backend provides an
|
||||
equivalent prefill implementation using per-request dense attention.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CPU_SDPA_MLA"
|
||||
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
# SDPA is always available on CPU.
|
||||
return True
|
||||
|
||||
def supports_out(self) -> bool:
|
||||
return True
|
||||
|
||||
def _ragged_sdpa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_cu_seq_lens: torch.Tensor,
|
||||
kv_cu_seq_lens: torch.Tensor,
|
||||
causal: bool,
|
||||
return_softmax_lse: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
# q/k/v: [num_tokens, num_heads, head_dim] (ragged, concatenated).
|
||||
# q_cu_seq_lens/kv_cu_seq_lens: [num_reqs + 1] row offsets for each
|
||||
# request's query and key/value tokens.
|
||||
num_reqs = q_cu_seq_lens.numel() - 1
|
||||
assert num_reqs == kv_cu_seq_lens.numel() - 1
|
||||
|
||||
out_chunks: list[torch.Tensor] = []
|
||||
lse_chunks: list[torch.Tensor] = []
|
||||
for r in range(num_reqs):
|
||||
q_start = int(q_cu_seq_lens[r])
|
||||
q_end = int(q_cu_seq_lens[r + 1])
|
||||
kv_start = int(kv_cu_seq_lens[r])
|
||||
kv_end = int(kv_cu_seq_lens[r + 1])
|
||||
|
||||
q_r = q[q_start:q_end].transpose(0, 1).float() # [NH, Sq, Hq]
|
||||
k_r = k[kv_start:kv_end].transpose(0, 1).float() # [NH, Sk, Hq]
|
||||
v_r = v[kv_start:kv_end].transpose(0, 1).float() # [NH, Sk, Hv]
|
||||
|
||||
attn_scores = torch.matmul(q_r, k_r.transpose(-2, -1)) * self.scale
|
||||
if causal:
|
||||
causal_mask = torch.ones(
|
||||
(q_end - q_start, kv_end - kv_start),
|
||||
dtype=torch.bool,
|
||||
device=attn_scores.device,
|
||||
).triu(1)
|
||||
attn_scores.masked_fill_(causal_mask.unsqueeze(0), float("-inf"))
|
||||
|
||||
attn_probs = torch.softmax(attn_scores, dim=-1, dtype=torch.float32)
|
||||
out_r = torch.matmul(attn_probs, v_r).transpose(0, 1).to(v.dtype)
|
||||
out_chunks.append(out_r) # [Sq, NH, Hv]
|
||||
if return_softmax_lse:
|
||||
lse_chunks.append(torch.logsumexp(attn_scores, dim=-1))
|
||||
|
||||
out = torch.cat(out_chunks, dim=0)
|
||||
if not return_softmax_lse:
|
||||
return out
|
||||
lse = torch.cat(lse_chunks, dim=1)
|
||||
return out, lse
|
||||
|
||||
def run_prefill_new_tokens(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
return_softmax_lse: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert output_scale is None, (
|
||||
"CPU MLA prefill backend does not support fused FP8 output"
|
||||
)
|
||||
out_t = self._ragged_sdpa(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
self._prefill_metadata.query_start_loc,
|
||||
self._prefill_metadata.query_start_loc,
|
||||
causal=True,
|
||||
return_softmax_lse=return_softmax_lse,
|
||||
)
|
||||
if out is not None and not return_softmax_lse:
|
||||
assert isinstance(out_t, torch.Tensor)
|
||||
out.copy_(out_t)
|
||||
return out
|
||||
return out_t
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
self,
|
||||
chunk: "MLACommonPrefillMetadata.ContextChunk",
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert out is None, "CPU MLA context prefill does not support in-place output"
|
||||
result = self._ragged_sdpa(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
chunk.query_start_loc,
|
||||
chunk.cu_seq_lens,
|
||||
causal=False,
|
||||
return_softmax_lse=True,
|
||||
)
|
||||
assert isinstance(result, tuple)
|
||||
out, lse = result
|
||||
return out, lse
|
||||
@@ -52,9 +52,7 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.mla.prefill.aiter_flash_attn."
|
||||
"AiterFlashAttnPrefillBackend"
|
||||
)
|
||||
CPU_NATIVE = (
|
||||
"vllm.v1.attention.backends.mla.prefill.cpu_native.CPUNativeMLAPrefillBackend"
|
||||
)
|
||||
CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend"
|
||||
# Placeholder for third-party/custom backends - must be registered before use
|
||||
# set to None to avoid alias with other backend, whose value is an empty string
|
||||
CUSTOM = None
|
||||
|
||||
@@ -107,10 +107,12 @@ def get_mla_prefill_backend(
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cpu():
|
||||
logger.info_once("Using CPU SDPA MLA prefill backend.")
|
||||
return MLAPrefillBackendEnum.CPU.get_class()
|
||||
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None:
|
||||
if current_platform.is_cpu():
|
||||
return MLAPrefillBackendEnum.CPU_NATIVE.get_class()
|
||||
logger.info_once(
|
||||
"Device capability not available, using FlashAttention MLA prefill backend."
|
||||
)
|
||||
|
||||
@@ -6,6 +6,68 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _merge_attn_states_torch(
|
||||
output: torch.Tensor,
|
||||
prefix_output: torch.Tensor,
|
||||
prefix_lse: torch.Tensor,
|
||||
suffix_output: torch.Tensor,
|
||||
suffix_lse: torch.Tensor,
|
||||
output_lse: torch.Tensor | None = None,
|
||||
prefill_tokens_with_context: int | None = None,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
assert output_scale is None, "CPU fallback does not support FP8 output"
|
||||
|
||||
num_tokens = output.shape[0]
|
||||
if prefill_tokens_with_context is None:
|
||||
prefill_tokens_with_context = num_tokens
|
||||
|
||||
prefix_lse = torch.where(
|
||||
torch.isinf(prefix_lse) & (prefix_lse > 0),
|
||||
torch.full_like(prefix_lse, float("-inf")),
|
||||
prefix_lse,
|
||||
)
|
||||
suffix_lse = torch.where(
|
||||
torch.isinf(suffix_lse) & (suffix_lse > 0),
|
||||
torch.full_like(suffix_lse, float("-inf")),
|
||||
suffix_lse,
|
||||
)
|
||||
|
||||
if prefill_tokens_with_context < num_tokens:
|
||||
output[prefill_tokens_with_context:].copy_(
|
||||
suffix_output[prefill_tokens_with_context:]
|
||||
)
|
||||
if output_lse is not None:
|
||||
output_lse[:, prefill_tokens_with_context:].copy_(
|
||||
suffix_lse[:, prefill_tokens_with_context:]
|
||||
)
|
||||
|
||||
if prefill_tokens_with_context == 0:
|
||||
return
|
||||
|
||||
prefix_lse = prefix_lse[:, :prefill_tokens_with_context].float()
|
||||
suffix_lse = suffix_lse[:, :prefill_tokens_with_context].float()
|
||||
max_lse = torch.maximum(prefix_lse, suffix_lse)
|
||||
prefix_se = torch.exp(prefix_lse - max_lse)
|
||||
suffix_se = torch.exp(suffix_lse - max_lse)
|
||||
out_se = prefix_se + suffix_se
|
||||
both_empty = max_lse == float("-inf")
|
||||
|
||||
prefix_scale = (prefix_se / out_se).transpose(0, 1).unsqueeze(-1)
|
||||
suffix_scale = (suffix_se / out_se).transpose(0, 1).unsqueeze(-1)
|
||||
merged = (
|
||||
prefix_output[:prefill_tokens_with_context].float() * prefix_scale
|
||||
+ suffix_output[:prefill_tokens_with_context].float() * suffix_scale
|
||||
)
|
||||
merged.masked_fill_(both_empty.transpose(0, 1).unsqueeze(-1), 0.0)
|
||||
output[:prefill_tokens_with_context].copy_(merged.to(output.dtype))
|
||||
|
||||
if output_lse is not None:
|
||||
merged_lse = torch.log(out_se) + max_lse
|
||||
merged_lse.masked_fill_(both_empty, float("-inf"))
|
||||
output_lse[:, :prefill_tokens_with_context].copy_(merged_lse)
|
||||
|
||||
|
||||
def merge_attn_states(
|
||||
output: torch.Tensor,
|
||||
prefix_output: torch.Tensor,
|
||||
@@ -94,6 +156,17 @@ def merge_attn_states(
|
||||
prefill_tokens_with_context,
|
||||
output_scale,
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
return _merge_attn_states_torch(
|
||||
output,
|
||||
prefix_output,
|
||||
prefix_lse,
|
||||
suffix_output,
|
||||
suffix_lse,
|
||||
output_lse,
|
||||
prefill_tokens_with_context,
|
||||
output_scale,
|
||||
)
|
||||
else:
|
||||
from vllm.v1.attention.ops.triton_merge_attn_states import (
|
||||
merge_attn_states,
|
||||
|
||||
Reference in New Issue
Block a user