[CPU][MLA] Fix prefill backend selection so MLA runs end-to-end on CPU (#51471)

Signed-off-by: baoloongmao <baoloongmao@tencent.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
maobaolong
2026-08-28 12:36:55 +08:00
committed by GitHub
co-authored by Codex
parent e6bfe03ad7
commit 6f7df92a8e
9 changed files with 442 additions and 69 deletions
+164 -2
View File
@@ -19,6 +19,51 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
CPUSDPAMLAPrefillBackend,
)
from vllm.v1.attention.backends.mla.prefill.selector import (
get_mla_prefill_backend,
)
def _reference_ragged_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
q_cu_seq_lens: torch.Tensor,
kv_cu_seq_lens: torch.Tensor,
*,
scale: float,
causal: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
outputs: list[torch.Tensor] = []
lses: list[torch.Tensor] = []
num_reqs = q_cu_seq_lens.numel() - 1
for req_idx in range(num_reqs):
q_start = int(q_cu_seq_lens[req_idx])
q_end = int(q_cu_seq_lens[req_idx + 1])
kv_start = int(kv_cu_seq_lens[req_idx])
kv_end = int(kv_cu_seq_lens[req_idx + 1])
q_r = q[q_start:q_end].transpose(0, 1).float()
k_r = k[kv_start:kv_end].transpose(0, 1).float()
v_r = v[kv_start:kv_end].transpose(0, 1).float()
scores = torch.matmul(q_r, k_r.transpose(-2, -1)) * scale
if causal:
causal_mask = torch.ones(
(q_end - q_start, kv_end - kv_start),
dtype=torch.bool,
device=scores.device,
).triu(1)
scores.masked_fill_(causal_mask.unsqueeze(0), float("-inf"))
probs = torch.softmax(scores, dim=-1)
outputs.append(torch.matmul(probs, v_r).transpose(0, 1).to(v.dtype))
lses.append(torch.logsumexp(scores, dim=-1))
return torch.cat(outputs, dim=0), torch.cat(lses, dim=1)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@@ -58,7 +103,7 @@ def test_kv_cache_cpu_write() -> None:
@pytest.mark.cpu_model
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_cpu_mla_backend_smoke() -> None:
def test_cpu_mla_backend_smoke(tmp_path) -> None:
# Shrink the architecture aggressively so the test is CI-friendly.
from vllm import LLM, SamplingParams
@@ -70,19 +115,136 @@ def test_cpu_mla_backend_smoke() -> None:
}
llm = LLM(
model="deepseek-ai/DeepSeek-V2-Lite",
skip_tokenizer_init=True,
trust_remote_code=True,
load_format="dummy",
enforce_eager=True,
max_model_len=128,
max_num_seqs=2,
block_size=16,
gpu_memory_utilization=0.25,
hf_overrides=hf_overrides,
kv_transfer_config={
"kv_connector": "ExampleConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"shared_storage_path": str(tmp_path),
},
},
)
sampling_params = SamplingParams(max_tokens=4, temperature=0.0)
outputs = llm.generate(["Hello", "MLA on CPU"], sampling_params)
outputs = llm.generate([[1, 2, 3, 4], [5, 6, 7]], sampling_params, use_tqdm=False)
assert len(outputs) == 2
for output in outputs:
# `dummy` weights do not produce meaningful text, but the number
# of generated tokens must match what we asked for.
assert len(output.outputs[0].token_ids) == 4
prefix_hit_prompt = [1] * 20
outputs = llm.generate([prefix_hit_prompt], sampling_params, use_tqdm=False)
assert len(outputs[0].outputs[0].token_ids) == 4
assert llm.reset_prefix_cache(reset_connector=False)
# ExampleConnector persists KV externally at block granularity. With a
# 20-token prompt and block_size=16, the second run reloads 16 cached
# tokens and forces the 4-token suffix through MLA prefill with context.
outputs = llm.generate([prefix_hit_prompt], sampling_params, use_tqdm=False)
assert len(outputs[0].outputs[0].token_ids) == 4
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_cpu_mla_prefill_backend_selected() -> None:
# On CPU (no device capability) the MLA prefill backend must be the
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
backend_cls = get_mla_prefill_backend(None)
assert backend_cls is CPUSDPAMLAPrefillBackend
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_cpu_mla_prefill_new_tokens() -> None:
# The SDPA prefill must exactly match a reference causal attention over
# the ragged (varlen) q/k/v layout, with per-request padding handled.
backend = CPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=0,
v_head_dim=16,
vllm_config=None,
)
class _PrefillMeta:
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
backend.prepare_metadata(_PrefillMeta())
q = torch.randn(6, 4, 16, dtype=torch.bfloat16)
k = torch.randn(6, 4, 16, dtype=torch.bfloat16)
v = torch.randn(6, 4, 16, dtype=torch.bfloat16)
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
ref, _ = _reference_ragged_attention(
q,
k,
v,
_PrefillMeta.query_start_loc,
_PrefillMeta.query_start_loc,
scale=0.25,
causal=True,
)
assert out.float().allclose(ref.float(), atol=2e-2)
assert out.shape == (6, 4, 16)
out_lse, lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
ref_out, ref_lse = _reference_ragged_attention(
q,
k,
v,
_PrefillMeta.query_start_loc,
_PrefillMeta.query_start_loc,
scale=0.25,
causal=True,
)
assert out_lse.float().allclose(ref_out.float(), atol=2e-2)
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
assert lse.shape == (4, 6)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
def test_cpu_mla_prefill_context_chunk() -> None:
backend = CPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=0,
v_head_dim=16,
vllm_config=None,
)
class _PrefillMeta:
query_start_loc = torch.tensor([0, 5], dtype=torch.int32)
class _Chunk:
query_start_loc = torch.tensor([0, 2, 5], dtype=torch.int32)
cu_seq_lens = torch.tensor([0, 3, 7], dtype=torch.int32)
backend.prepare_metadata(_PrefillMeta())
q = torch.randn(5, 4, 16, dtype=torch.bfloat16)
k = torch.randn(7, 4, 16, dtype=torch.bfloat16)
v = torch.randn(7, 4, 16, dtype=torch.bfloat16)
out, lse = backend.run_prefill_context_chunk(_Chunk(), q, k, v)
ref_out, ref_lse = _reference_ragged_attention(
q,
k,
v,
_Chunk.query_start_loc,
_Chunk.cu_seq_lens,
scale=0.25,
causal=False,
)
assert out.float().allclose(ref_out.float(), atol=2e-2)
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
assert out.shape == (5, 4, 16)
assert lse.shape == (4, 5)
@@ -58,10 +58,31 @@ def _make_vllm_config(
class TestGetMLAPrefillBackend:
"""Tests for get_mla_prefill_backend (public API)."""
def test_no_device_capability_returns_flash_attn(self):
def test_cpu_uses_sdpa_prefill(self):
vllm_config = _make_vllm_config()
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is MLAPrefillBackendEnum.CPU.get_class()
def test_no_device_capability_returns_flash_attn(self):
vllm_config = _make_vllm_config()
class FlashAttnBackend:
@staticmethod
def get_name():
return "FLASH_ATTN"
with (
patch("vllm.platforms.current_platform") as mock_platform,
patch.object(
MLAPrefillBackendEnum.FLASH_ATTN,
"get_class",
return_value=FlashAttnBackend,
),
):
mock_platform.get_device_capability.return_value = None
mock_platform.is_cpu.return_value = False
@@ -80,6 +101,7 @@ class TestGetMLAPrefillBackend:
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = False
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)
@@ -98,6 +120,7 @@ class TestGetMLAPrefillBackend:
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = False
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)
@@ -111,6 +134,7 @@ class TestGetMLAPrefillBackend:
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = False
mock_platform.get_device_capability.return_value = DeviceCapability(
major=10, minor=0
)
@@ -153,6 +177,7 @@ class TestGetMLAPrefillBackend:
return_value=3,
),
):
mock_platform.is_cpu.return_value = False
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)