[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell (#41778)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Yongye Zhu
2026-05-13 23:48:02 -07:00
committed by GitHub
co-authored by Roger Wang
parent fd7d858c8a
commit 0d2732dd91
14 changed files with 639 additions and 88 deletions
+21 -12
View File
@@ -224,19 +224,28 @@ def init_test_http_connection():
def dist_init():
from tests.utils import ensure_current_vllm_config
temp_file = tempfile.mkstemp()[1]
# Close the fd returned by mkstemp; FileStore opens the path itself.
# Leaving it open leaks one FD per test and eventually exhausts the
# ulimit, causing FileStore's destructor to throw c10::DistStoreError
# ("Too many open files") during gc and abort the process.
fd, temp_file = tempfile.mkstemp()
os.close(fd)
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend="nccl",
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
try:
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend="nccl",
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
finally:
with contextlib.suppress(OSError):
os.unlink(temp_file)
@pytest.fixture
+66 -7
View File
@@ -20,17 +20,20 @@ from tests.v1.attention.utils import (
from vllm import _custom_ops as ops
from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import (
MLAAttention,
QueryLenSupport,
_DecodeConcatQuantFP8,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.backends.mla.prefill import (
MLAPrefillBackendEnum,
get_mla_prefill_backend,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import MLAAttentionSpec
@@ -41,6 +44,7 @@ BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN_MLA,
AttentionBackendEnum.FLASHINFER_MLA,
AttentionBackendEnum.TRITON_MLA,
AttentionBackendEnum.TOKENSPEED_MLA,
]
DEVICE_TYPE = current_platform.device_type
@@ -49,6 +53,7 @@ DEVICE_TYPE = current_platform.device_type
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHINFER_MLA)
BACKENDS_TO_TEST.remove(AttentionBackendEnum.TOKENSPEED_MLA)
# Remove FLASH_ATTN_MLA from the list if not supported
if not flash_attn_supports_mla():
@@ -58,6 +63,22 @@ if not flash_attn_supports_mla():
if not is_flashmla_dense_supported()[0]:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHMLA)
# Remove TOKENSPEED_MLA if the optional package is not installed
if AttentionBackendEnum.TOKENSPEED_MLA in BACKENDS_TO_TEST:
try:
import tokenspeed_mla # noqa: F401
except ImportError:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.TOKENSPEED_MLA)
# Filtered per-test via validate_configuration (capability/deps/dims).
PREFILL_BACKENDS_TO_TEST = [
MLAPrefillBackendEnum.FLASH_ATTN,
MLAPrefillBackendEnum.FLASHINFER,
MLAPrefillBackendEnum.TRTLLM_RAGGED,
MLAPrefillBackendEnum.TOKENSPEED_MLA,
]
SPEC_DECODE_BACKENDS = []
for backend in BACKENDS_TO_TEST:
@@ -389,14 +410,18 @@ class MockSparseMLAAttentionLayer:
return output
class MockMLAAttentionLayer(AttentionLayerBase):
class MockMLAAttentionLayer(MLAAttention):
"""A mock MLA attention layer for testing.
This replicates the forward_impl logic from MLAAttention to allow
testing MLA backends without the full layer infrastructure.
The W_UK_T and W_UV weight matrices are created on the layer (like in
MLAAttention.process_weights_after_loading), not on the impl.
Subclasses MLAAttention so that backends that filter
`static_forward_context` by `isinstance(layer, MLAAttention)` (e.g.
FlashInfer prefill, which reads sm_scale through that filter) see the
mock as a real MLA layer. MLAAttention.__init__ is intentionally
skipped — it would create its own impl/prefill_backend and self-register
in static_forward_context, which fights what the test sets up below.
"""
def __init__(
@@ -412,6 +437,7 @@ class MockMLAAttentionLayer(AttentionLayerBase):
q_scale: float,
k_scale: float,
):
torch.nn.Module.__init__(self)
self.impl = impl
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
@@ -562,11 +588,15 @@ def run_attention_backend(
q_scale: float,
k_scale: float,
kv_cache_dtype: str = "auto",
prefill_backend: MLAPrefillBackendEnum | None = None,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
builder_cls, impl_cls = try_get_attention_backend(backend)
# Force the prefill backend selection (None means auto-select).
vllm_config.attention_config.mla_prefill_backend = prefill_backend
# Set the current vllm config so that get_current_vllm_config() works
# in the backend implementations
with set_current_vllm_config(vllm_config):
@@ -578,7 +608,11 @@ def run_attention_backend(
vllm_config.parallel_config
)
head_size = vllm_config.model_config.get_head_size()
scale = 1.0 / (head_size**0.5)
# Production MLA passes 1/sqrt(qk_head_dim) (the prefill scale) to the
# impl and forwards the same value to the prefill backend. FLASHINFER
# prefill reads sm_scale back from impl.scale via global_hyperparameters
# at plan() time, so impl.scale must agree with prefill_backend.scale.
scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5
impl = impl_cls(
num_heads=num_heads,
head_size=head_size,
@@ -683,6 +717,7 @@ def run_attention_backend(
@pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@pytest.mark.parametrize("prefill_backend", PREFILL_BACKENDS_TO_TEST)
def test_backend_correctness(
default_vllm_config,
dist_init,
@@ -693,6 +728,7 @@ def test_backend_correctness(
kv_cache_dtype: str,
q_scale: float,
k_scale: float,
prefill_backend: MLAPrefillBackendEnum,
):
"""
Test that all backends produce similar outputs to a reference implementation
@@ -729,6 +765,24 @@ def test_backend_correctness(
if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
# Skip prefill backends that can't satisfy capability/deps/R1 constraints.
from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig,
)
try:
prefill_invalid_reasons = prefill_backend.get_class().validate_configuration(
current_platform.get_device_capability(),
MLAPrefillSelectorConfig(dtype=torch.bfloat16, is_r1_compatible=True),
)
except ImportError:
prefill_invalid_reasons = ["ImportError"]
if prefill_invalid_reasons:
pytest.skip(
f"Prefill backend {prefill_backend.name} unavailable: "
f"{prefill_invalid_reasons}"
)
batch_spec = BATCH_SPECS[batch_spec_name]
is_spec_decode_test = batch_spec_name.startswith("spec_decode")
unique_block_sizes = sorted(set(BACKEND_BLOCK_SIZES[b] for b in backends_to_test))
@@ -799,9 +853,13 @@ def test_backend_correctness(
assert kv_lora_rank + qk_rope_head_dim == head_size, (
f"MLA dimensions don't match: {total_head_size} != {head_size}"
)
decode_scale = 1.0 / (total_head_size**0.5)
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
prefill_scale = qk_head_dim**-0.5
# MLA reuses prefill_scale for the decode path: production sets
# impl.scale = 1/sqrt(qk_head_dim) and the decode kernels apply it even
# though the latent attention runs at head_size dimensions. Keeping the
# reference here in sync with run_attention_backend's impl.scale.
decode_scale = prefill_scale
# 2. Generate data and compute SDPA reference output for MLA
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
@@ -1092,6 +1150,7 @@ def test_backend_correctness(
qk_rope_head_dim,
v_head_dim,
mock_kv_b_proj,
prefill_backend=prefill_backend,
q_scale=q_scale,
k_scale=k_scale,
kv_cache_dtype=kv_cache_dtype,