[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell (#41778)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Yongye Zhu
2026-05-13 23:48:02 -07:00
committed by GitHub
co-authored by Roger Wang
parent fd7d858c8a
commit 0d2732dd91
14 changed files with 639 additions and 88 deletions
@@ -53,6 +53,7 @@ backends:
- FLASHINFER_MLA
- FLASH_ATTN_MLA # Hopper only
- FLASHMLA # Hopper only
- TOKENSPEED_MLA # Blackwell + R1 dims + FP8 KV (use --kv-cache-dtype fp8)
device: "cuda:0"
repeats: 100
@@ -3,6 +3,7 @@
# Compares all available MLA prefill backends:
# FA backends: fa2, fa3, fa4 (FlashAttention versions)
# Non-FA: flashinfer, cudnn, trtllm (Blackwell-only, require flashinfer)
# CuTe DSL: tokenspeed (Blackwell + R1 dims, requires tokenspeed_mla)
#
# Uses cutlass_mla as the decode backend for impl construction
# (only the prefill path is exercised).
@@ -120,6 +121,7 @@ prefill_backends:
- flashinfer
- cudnn
- trtllm
- tokenspeed
device: "cuda:0"
repeats: 20
+67 -63
View File
@@ -179,19 +179,27 @@ def create_minimal_vllm_config(
if prefill_backend is not None:
prefill_cfg = get_prefill_backend_config(prefill_backend)
if prefill_cfg["flash_attn_version"] is not None:
vllm_config.attention_config.flash_attn_version = prefill_cfg[
"flash_attn_version"
if prefill_cfg.get("mla_prefill_backend_enum") is not None:
# Registry-based backends bypass the deprecated boolean flags.
from vllm.v1.attention.backends.mla.prefill import MLAPrefillBackendEnum
vllm_config.attention_config.mla_prefill_backend = MLAPrefillBackendEnum[
prefill_cfg["mla_prefill_backend_enum"]
]
vllm_config.attention_config.disable_flashinfer_prefill = prefill_cfg[
"disable_flashinfer_prefill"
]
vllm_config.attention_config.use_cudnn_prefill = prefill_cfg[
"use_cudnn_prefill"
]
vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill = prefill_cfg[
"use_trtllm_ragged_deepseek_prefill"
]
else:
if prefill_cfg["flash_attn_version"] is not None:
vllm_config.attention_config.flash_attn_version = prefill_cfg[
"flash_attn_version"
]
vllm_config.attention_config.disable_flashinfer_prefill = prefill_cfg[
"disable_flashinfer_prefill"
]
vllm_config.attention_config.use_cudnn_prefill = prefill_cfg[
"use_cudnn_prefill"
]
vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill = (
prefill_cfg["use_trtllm_ragged_deepseek_prefill"]
)
return vllm_config
@@ -223,22 +231,17 @@ _PREFILL_BACKEND_CONFIG: dict[str, dict] = {
"use_trtllm_ragged_deepseek_prefill": False,
},
"flashinfer": {
"flash_attn_version": None,
"disable_flashinfer_prefill": False,
"use_cudnn_prefill": False,
"use_trtllm_ragged_deepseek_prefill": False,
"mla_prefill_backend_enum": "FLASHINFER",
},
"cudnn": {
"flash_attn_version": None,
"disable_flashinfer_prefill": True,
"use_cudnn_prefill": True,
"use_trtllm_ragged_deepseek_prefill": False,
# cuDNN prefill backend was removed; AttentionConfig raises on use.
"mla_prefill_backend_enum": "FLASHINFER",
},
"trtllm": {
"flash_attn_version": None,
"disable_flashinfer_prefill": True,
"use_cudnn_prefill": False,
"use_trtllm_ragged_deepseek_prefill": True,
"mla_prefill_backend_enum": "TRTLLM_RAGGED",
},
"tokenspeed": {
"mla_prefill_backend_enum": "TOKENSPEED_MLA",
},
}
@@ -625,6 +628,21 @@ def _create_backend_impl(
# Create mock layer
layer = MockLayer(device, impl=impl, kv_cache_spec=kv_cache_spec)
# Attach a prefill backend (MLAAttention does this in __init__; the metadata
# builder reads layer.prefill_backend from static_forward_context).
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
layer.prefill_backend = prefill_backend_cls(
num_heads=mla_dims["num_q_heads"],
scale=(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"]) ** -0.5,
kv_lora_rank=mla_dims["kv_lora_rank"],
qk_nope_head_dim=mla_dims["qk_nope_head_dim"],
qk_rope_head_dim=mla_dims["qk_rope_head_dim"],
v_head_dim=mla_dims["v_head_dim"],
vllm_config=vllm_config,
)
# Create builder instance if needed
builder_instance = None
if builder_class:
@@ -961,19 +979,6 @@ def _run_mla_benchmark_batched(
results = []
with set_current_vllm_config(vllm_config):
# Clear cached prefill backend detection functions so they re-evaluate
# with the current VllmConfig. These are @functools.cache decorated and
# would otherwise return stale results from a previous backend's config.
from vllm.model_executor.layers.attention.mla_attention import (
use_cudnn_prefill,
use_flashinfer_prefill,
use_trtllm_ragged_deepseek_prefill,
)
use_flashinfer_prefill.cache_clear()
use_cudnn_prefill.cache_clear()
use_trtllm_ragged_deepseek_prefill.cache_clear()
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
impl, layer, builder_instance, indexer = _create_backend_impl(
backend_cfg,
@@ -985,36 +990,35 @@ def _run_mla_benchmark_batched(
kv_cache_dtype=kv_cache_dtype,
)
# Verify the actual prefill backend matches what was requested
# Verify the actual prefill backend matches what was requested. The
# selector + impl construction already raise on misuse; here we just
# check the resolved class against the requested name as a sanity guard.
if prefill_backend is not None:
prefill_cfg = get_prefill_backend_config(prefill_backend)
fa_version = prefill_cfg["flash_attn_version"]
if fa_version is not None:
# FA backend: verify the impl's FA version
actual_fa_version = getattr(impl, "vllm_flash_attn_version", None)
expected_class = {
"fa2": "FlashAttnPrefillBackend",
"fa3": "FlashAttnPrefillBackend",
"fa4": "FlashAttnPrefillBackend",
"flashinfer": "FlashInferPrefillBackend",
"trtllm": "TrtllmRaggedPrefillBackend",
"tokenspeed": "TokenspeedMLAPrefillBackend",
}.get(prefill_backend)
actual_class = type(getattr(layer, "prefill_backend", None)).__name__
if expected_class and actual_class != expected_class:
raise RuntimeError(
f"Prefill backend '{prefill_backend}' requested "
f"{expected_class}, got {actual_class}. Check "
f"attention_config plumbing or installed deps."
)
if prefill_backend in {"fa2", "fa3", "fa4"}:
fa_version = int(prefill_backend[2:])
actual_fa_version = getattr(
layer.prefill_backend, "vllm_flash_attn_version", None
)
if actual_fa_version != fa_version:
raise RuntimeError(
f"Prefill backend '{prefill_backend}' requested FA "
f"version {fa_version}, but the impl is using FA "
f"version {actual_fa_version}. Check "
f"vllm/v1/attention/backends/fa_utils.py."
)
else:
# Non-FA backend: verify the builder picked the right path
expected_flags = {
"flashinfer": "_use_fi_prefill",
"cudnn": "_use_cudnn_prefill",
"trtllm": "_use_trtllm_ragged_prefill",
}
flag_name = expected_flags.get(prefill_backend)
if flag_name and not getattr(builder_instance, flag_name, False):
raise RuntimeError(
f"Prefill backend '{prefill_backend}' was requested "
f"but the metadata builder did not enable it. This "
f"usually means a dependency is missing (e.g., "
f"flashinfer not installed) or the platform doesn't "
f"support it."
f"version {fa_version}, got "
f"{actual_fa_version} on {actual_class}."
)
# Run each benchmark with the shared impl