[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell (#41778)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -53,6 +53,7 @@ backends:
|
||||
- FLASHINFER_MLA
|
||||
- FLASH_ATTN_MLA # Hopper only
|
||||
- FLASHMLA # Hopper only
|
||||
- TOKENSPEED_MLA # Blackwell + R1 dims + FP8 KV (use --kv-cache-dtype fp8)
|
||||
|
||||
device: "cuda:0"
|
||||
repeats: 100
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# Compares all available MLA prefill backends:
|
||||
# FA backends: fa2, fa3, fa4 (FlashAttention versions)
|
||||
# Non-FA: flashinfer, cudnn, trtllm (Blackwell-only, require flashinfer)
|
||||
# CuTe DSL: tokenspeed (Blackwell + R1 dims, requires tokenspeed_mla)
|
||||
#
|
||||
# Uses cutlass_mla as the decode backend for impl construction
|
||||
# (only the prefill path is exercised).
|
||||
@@ -120,6 +121,7 @@ prefill_backends:
|
||||
- flashinfer
|
||||
- cudnn
|
||||
- trtllm
|
||||
- tokenspeed
|
||||
|
||||
device: "cuda:0"
|
||||
repeats: 20
|
||||
|
||||
@@ -179,19 +179,27 @@ def create_minimal_vllm_config(
|
||||
|
||||
if prefill_backend is not None:
|
||||
prefill_cfg = get_prefill_backend_config(prefill_backend)
|
||||
if prefill_cfg["flash_attn_version"] is not None:
|
||||
vllm_config.attention_config.flash_attn_version = prefill_cfg[
|
||||
"flash_attn_version"
|
||||
if prefill_cfg.get("mla_prefill_backend_enum") is not None:
|
||||
# Registry-based backends bypass the deprecated boolean flags.
|
||||
from vllm.v1.attention.backends.mla.prefill import MLAPrefillBackendEnum
|
||||
|
||||
vllm_config.attention_config.mla_prefill_backend = MLAPrefillBackendEnum[
|
||||
prefill_cfg["mla_prefill_backend_enum"]
|
||||
]
|
||||
vllm_config.attention_config.disable_flashinfer_prefill = prefill_cfg[
|
||||
"disable_flashinfer_prefill"
|
||||
]
|
||||
vllm_config.attention_config.use_cudnn_prefill = prefill_cfg[
|
||||
"use_cudnn_prefill"
|
||||
]
|
||||
vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill = prefill_cfg[
|
||||
"use_trtllm_ragged_deepseek_prefill"
|
||||
]
|
||||
else:
|
||||
if prefill_cfg["flash_attn_version"] is not None:
|
||||
vllm_config.attention_config.flash_attn_version = prefill_cfg[
|
||||
"flash_attn_version"
|
||||
]
|
||||
vllm_config.attention_config.disable_flashinfer_prefill = prefill_cfg[
|
||||
"disable_flashinfer_prefill"
|
||||
]
|
||||
vllm_config.attention_config.use_cudnn_prefill = prefill_cfg[
|
||||
"use_cudnn_prefill"
|
||||
]
|
||||
vllm_config.attention_config.use_trtllm_ragged_deepseek_prefill = (
|
||||
prefill_cfg["use_trtllm_ragged_deepseek_prefill"]
|
||||
)
|
||||
|
||||
return vllm_config
|
||||
|
||||
@@ -223,22 +231,17 @@ _PREFILL_BACKEND_CONFIG: dict[str, dict] = {
|
||||
"use_trtllm_ragged_deepseek_prefill": False,
|
||||
},
|
||||
"flashinfer": {
|
||||
"flash_attn_version": None,
|
||||
"disable_flashinfer_prefill": False,
|
||||
"use_cudnn_prefill": False,
|
||||
"use_trtllm_ragged_deepseek_prefill": False,
|
||||
"mla_prefill_backend_enum": "FLASHINFER",
|
||||
},
|
||||
"cudnn": {
|
||||
"flash_attn_version": None,
|
||||
"disable_flashinfer_prefill": True,
|
||||
"use_cudnn_prefill": True,
|
||||
"use_trtllm_ragged_deepseek_prefill": False,
|
||||
# cuDNN prefill backend was removed; AttentionConfig raises on use.
|
||||
"mla_prefill_backend_enum": "FLASHINFER",
|
||||
},
|
||||
"trtllm": {
|
||||
"flash_attn_version": None,
|
||||
"disable_flashinfer_prefill": True,
|
||||
"use_cudnn_prefill": False,
|
||||
"use_trtllm_ragged_deepseek_prefill": True,
|
||||
"mla_prefill_backend_enum": "TRTLLM_RAGGED",
|
||||
},
|
||||
"tokenspeed": {
|
||||
"mla_prefill_backend_enum": "TOKENSPEED_MLA",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -625,6 +628,21 @@ def _create_backend_impl(
|
||||
# Create mock layer
|
||||
layer = MockLayer(device, impl=impl, kv_cache_spec=kv_cache_spec)
|
||||
|
||||
# Attach a prefill backend (MLAAttention does this in __init__; the metadata
|
||||
# builder reads layer.prefill_backend from static_forward_context).
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
layer.prefill_backend = prefill_backend_cls(
|
||||
num_heads=mla_dims["num_q_heads"],
|
||||
scale=(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"]) ** -0.5,
|
||||
kv_lora_rank=mla_dims["kv_lora_rank"],
|
||||
qk_nope_head_dim=mla_dims["qk_nope_head_dim"],
|
||||
qk_rope_head_dim=mla_dims["qk_rope_head_dim"],
|
||||
v_head_dim=mla_dims["v_head_dim"],
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
# Create builder instance if needed
|
||||
builder_instance = None
|
||||
if builder_class:
|
||||
@@ -961,19 +979,6 @@ def _run_mla_benchmark_batched(
|
||||
results = []
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
# Clear cached prefill backend detection functions so they re-evaluate
|
||||
# with the current VllmConfig. These are @functools.cache decorated and
|
||||
# would otherwise return stale results from a previous backend's config.
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
use_cudnn_prefill,
|
||||
use_flashinfer_prefill,
|
||||
use_trtllm_ragged_deepseek_prefill,
|
||||
)
|
||||
|
||||
use_flashinfer_prefill.cache_clear()
|
||||
use_cudnn_prefill.cache_clear()
|
||||
use_trtllm_ragged_deepseek_prefill.cache_clear()
|
||||
|
||||
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
|
||||
impl, layer, builder_instance, indexer = _create_backend_impl(
|
||||
backend_cfg,
|
||||
@@ -985,36 +990,35 @@ def _run_mla_benchmark_batched(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
# Verify the actual prefill backend matches what was requested
|
||||
# Verify the actual prefill backend matches what was requested. The
|
||||
# selector + impl construction already raise on misuse; here we just
|
||||
# check the resolved class against the requested name as a sanity guard.
|
||||
if prefill_backend is not None:
|
||||
prefill_cfg = get_prefill_backend_config(prefill_backend)
|
||||
fa_version = prefill_cfg["flash_attn_version"]
|
||||
|
||||
if fa_version is not None:
|
||||
# FA backend: verify the impl's FA version
|
||||
actual_fa_version = getattr(impl, "vllm_flash_attn_version", None)
|
||||
expected_class = {
|
||||
"fa2": "FlashAttnPrefillBackend",
|
||||
"fa3": "FlashAttnPrefillBackend",
|
||||
"fa4": "FlashAttnPrefillBackend",
|
||||
"flashinfer": "FlashInferPrefillBackend",
|
||||
"trtllm": "TrtllmRaggedPrefillBackend",
|
||||
"tokenspeed": "TokenspeedMLAPrefillBackend",
|
||||
}.get(prefill_backend)
|
||||
actual_class = type(getattr(layer, "prefill_backend", None)).__name__
|
||||
if expected_class and actual_class != expected_class:
|
||||
raise RuntimeError(
|
||||
f"Prefill backend '{prefill_backend}' requested "
|
||||
f"{expected_class}, got {actual_class}. Check "
|
||||
f"attention_config plumbing or installed deps."
|
||||
)
|
||||
if prefill_backend in {"fa2", "fa3", "fa4"}:
|
||||
fa_version = int(prefill_backend[2:])
|
||||
actual_fa_version = getattr(
|
||||
layer.prefill_backend, "vllm_flash_attn_version", None
|
||||
)
|
||||
if actual_fa_version != fa_version:
|
||||
raise RuntimeError(
|
||||
f"Prefill backend '{prefill_backend}' requested FA "
|
||||
f"version {fa_version}, but the impl is using FA "
|
||||
f"version {actual_fa_version}. Check "
|
||||
f"vllm/v1/attention/backends/fa_utils.py."
|
||||
)
|
||||
else:
|
||||
# Non-FA backend: verify the builder picked the right path
|
||||
expected_flags = {
|
||||
"flashinfer": "_use_fi_prefill",
|
||||
"cudnn": "_use_cudnn_prefill",
|
||||
"trtllm": "_use_trtllm_ragged_prefill",
|
||||
}
|
||||
flag_name = expected_flags.get(prefill_backend)
|
||||
if flag_name and not getattr(builder_instance, flag_name, False):
|
||||
raise RuntimeError(
|
||||
f"Prefill backend '{prefill_backend}' was requested "
|
||||
f"but the metadata builder did not enable it. This "
|
||||
f"usually means a dependency is missing (e.g., "
|
||||
f"flashinfer not installed) or the platform doesn't "
|
||||
f"support it."
|
||||
f"version {fa_version}, got "
|
||||
f"{actual_fa_version} on {actual_class}."
|
||||
)
|
||||
|
||||
# Run each benchmark with the shared impl
|
||||
|
||||
Reference in New Issue
Block a user