[CPU][MLA] Fix prefill backend selection so MLA runs end-to-end on CPU (#51471)
Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -19,6 +19,51 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
|
||||
CPUSDPAMLAPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
get_mla_prefill_backend,
|
||||
)
|
||||
|
||||
|
||||
def _reference_ragged_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
q_cu_seq_lens: torch.Tensor,
|
||||
kv_cu_seq_lens: torch.Tensor,
|
||||
*,
|
||||
scale: float,
|
||||
causal: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
outputs: list[torch.Tensor] = []
|
||||
lses: list[torch.Tensor] = []
|
||||
num_reqs = q_cu_seq_lens.numel() - 1
|
||||
for req_idx in range(num_reqs):
|
||||
q_start = int(q_cu_seq_lens[req_idx])
|
||||
q_end = int(q_cu_seq_lens[req_idx + 1])
|
||||
kv_start = int(kv_cu_seq_lens[req_idx])
|
||||
kv_end = int(kv_cu_seq_lens[req_idx + 1])
|
||||
|
||||
q_r = q[q_start:q_end].transpose(0, 1).float()
|
||||
k_r = k[kv_start:kv_end].transpose(0, 1).float()
|
||||
v_r = v[kv_start:kv_end].transpose(0, 1).float()
|
||||
|
||||
scores = torch.matmul(q_r, k_r.transpose(-2, -1)) * scale
|
||||
if causal:
|
||||
causal_mask = torch.ones(
|
||||
(q_end - q_start, kv_end - kv_start),
|
||||
dtype=torch.bool,
|
||||
device=scores.device,
|
||||
).triu(1)
|
||||
scores.masked_fill_(causal_mask.unsqueeze(0), float("-inf"))
|
||||
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
outputs.append(torch.matmul(probs, v_r).transpose(0, 1).to(v.dtype))
|
||||
lses.append(torch.logsumexp(scores, dim=-1))
|
||||
|
||||
return torch.cat(outputs, dim=0), torch.cat(lses, dim=1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
@@ -58,7 +103,7 @@ def test_kv_cache_cpu_write() -> None:
|
||||
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_cpu_mla_backend_smoke() -> None:
|
||||
def test_cpu_mla_backend_smoke(tmp_path) -> None:
|
||||
# Shrink the architecture aggressively so the test is CI-friendly.
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
@@ -70,19 +115,136 @@ def test_cpu_mla_backend_smoke() -> None:
|
||||
}
|
||||
llm = LLM(
|
||||
model="deepseek-ai/DeepSeek-V2-Lite",
|
||||
skip_tokenizer_init=True,
|
||||
trust_remote_code=True,
|
||||
load_format="dummy",
|
||||
enforce_eager=True,
|
||||
max_model_len=128,
|
||||
max_num_seqs=2,
|
||||
block_size=16,
|
||||
gpu_memory_utilization=0.25,
|
||||
hf_overrides=hf_overrides,
|
||||
kv_transfer_config={
|
||||
"kv_connector": "ExampleConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"shared_storage_path": str(tmp_path),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=4, temperature=0.0)
|
||||
outputs = llm.generate(["Hello", "MLA on CPU"], sampling_params)
|
||||
outputs = llm.generate([[1, 2, 3, 4], [5, 6, 7]], sampling_params, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for output in outputs:
|
||||
# `dummy` weights do not produce meaningful text, but the number
|
||||
# of generated tokens must match what we asked for.
|
||||
assert len(output.outputs[0].token_ids) == 4
|
||||
|
||||
prefix_hit_prompt = [1] * 20
|
||||
outputs = llm.generate([prefix_hit_prompt], sampling_params, use_tqdm=False)
|
||||
assert len(outputs[0].outputs[0].token_ids) == 4
|
||||
assert llm.reset_prefix_cache(reset_connector=False)
|
||||
|
||||
# ExampleConnector persists KV externally at block granularity. With a
|
||||
# 20-token prompt and block_size=16, the second run reloads 16 cached
|
||||
# tokens and forces the 4-token suffix through MLA prefill with context.
|
||||
outputs = llm.generate([prefix_hit_prompt], sampling_params, use_tqdm=False)
|
||||
assert len(outputs[0].outputs[0].token_ids) == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_cpu_mla_prefill_backend_selected() -> None:
|
||||
# On CPU (no device capability) the MLA prefill backend must be the
|
||||
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
|
||||
backend_cls = get_mla_prefill_backend(None)
|
||||
assert backend_cls is CPUSDPAMLAPrefillBackend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_cpu_mla_prefill_new_tokens() -> None:
|
||||
# The SDPA prefill must exactly match a reference causal attention over
|
||||
# the ragged (varlen) q/k/v layout, with per-request padding handled.
|
||||
backend = CPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=0,
|
||||
v_head_dim=16,
|
||||
vllm_config=None,
|
||||
)
|
||||
|
||||
class _PrefillMeta:
|
||||
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
|
||||
backend.prepare_metadata(_PrefillMeta())
|
||||
q = torch.randn(6, 4, 16, dtype=torch.bfloat16)
|
||||
k = torch.randn(6, 4, 16, dtype=torch.bfloat16)
|
||||
v = torch.randn(6, 4, 16, dtype=torch.bfloat16)
|
||||
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
|
||||
ref, _ = _reference_ragged_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_PrefillMeta.query_start_loc,
|
||||
_PrefillMeta.query_start_loc,
|
||||
scale=0.25,
|
||||
causal=True,
|
||||
)
|
||||
assert out.float().allclose(ref.float(), atol=2e-2)
|
||||
assert out.shape == (6, 4, 16)
|
||||
|
||||
out_lse, lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
|
||||
ref_out, ref_lse = _reference_ragged_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_PrefillMeta.query_start_loc,
|
||||
_PrefillMeta.query_start_loc,
|
||||
scale=0.25,
|
||||
causal=True,
|
||||
)
|
||||
assert out_lse.float().allclose(ref_out.float(), atol=2e-2)
|
||||
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
|
||||
assert lse.shape == (4, 6)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
def test_cpu_mla_prefill_context_chunk() -> None:
|
||||
backend = CPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=0,
|
||||
v_head_dim=16,
|
||||
vllm_config=None,
|
||||
)
|
||||
|
||||
class _PrefillMeta:
|
||||
query_start_loc = torch.tensor([0, 5], dtype=torch.int32)
|
||||
|
||||
class _Chunk:
|
||||
query_start_loc = torch.tensor([0, 2, 5], dtype=torch.int32)
|
||||
cu_seq_lens = torch.tensor([0, 3, 7], dtype=torch.int32)
|
||||
|
||||
backend.prepare_metadata(_PrefillMeta())
|
||||
q = torch.randn(5, 4, 16, dtype=torch.bfloat16)
|
||||
k = torch.randn(7, 4, 16, dtype=torch.bfloat16)
|
||||
v = torch.randn(7, 4, 16, dtype=torch.bfloat16)
|
||||
|
||||
out, lse = backend.run_prefill_context_chunk(_Chunk(), q, k, v)
|
||||
ref_out, ref_lse = _reference_ragged_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_Chunk.query_start_loc,
|
||||
_Chunk.cu_seq_lens,
|
||||
scale=0.25,
|
||||
causal=False,
|
||||
)
|
||||
assert out.float().allclose(ref_out.float(), atol=2e-2)
|
||||
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
|
||||
assert out.shape == (5, 4, 16)
|
||||
assert lse.shape == (4, 5)
|
||||
|
||||
@@ -58,10 +58,31 @@ def _make_vllm_config(
|
||||
class TestGetMLAPrefillBackend:
|
||||
"""Tests for get_mla_prefill_backend (public API)."""
|
||||
|
||||
def test_no_device_capability_returns_flash_attn(self):
|
||||
def test_cpu_uses_sdpa_prefill(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is MLAPrefillBackendEnum.CPU.get_class()
|
||||
|
||||
def test_no_device_capability_returns_flash_attn(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
class FlashAttnBackend:
|
||||
@staticmethod
|
||||
def get_name():
|
||||
return "FLASH_ATTN"
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
"get_class",
|
||||
return_value=FlashAttnBackend,
|
||||
),
|
||||
):
|
||||
mock_platform.get_device_capability.return_value = None
|
||||
mock_platform.is_cpu.return_value = False
|
||||
|
||||
@@ -80,6 +101,7 @@ class TestGetMLAPrefillBackend:
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.get_device_capability.return_value = DeviceCapability(
|
||||
major=9, minor=0
|
||||
)
|
||||
@@ -98,6 +120,7 @@ class TestGetMLAPrefillBackend:
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.get_device_capability.return_value = DeviceCapability(
|
||||
major=9, minor=0
|
||||
)
|
||||
@@ -111,6 +134,7 @@ class TestGetMLAPrefillBackend:
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.get_device_capability.return_value = DeviceCapability(
|
||||
major=10, minor=0
|
||||
)
|
||||
@@ -153,6 +177,7 @@ class TestGetMLAPrefillBackend:
|
||||
return_value=3,
|
||||
),
|
||||
):
|
||||
mock_platform.is_cpu.return_value = False
|
||||
mock_platform.get_device_capability.return_value = DeviceCapability(
|
||||
major=9, minor=0
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user