[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell (#41778)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Yongye Zhu
2026-05-13 23:48:02 -07:00
committed by GitHub
co-authored by Roger Wang
parent fd7d858c8a
commit 0d2732dd91
14 changed files with 639 additions and 88 deletions
@@ -1362,6 +1362,7 @@ def backend_supports_prefill_query_quantization() -> bool:
return backend_cls.get_name() in (
"FLASHINFER",
"TRTLLM_RAGGED",
"TOKENSPEED_MLA",
)
+4
View File
@@ -110,6 +110,10 @@ def _get_backend_priorities(
return [
AttentionBackendEnum.FLASHINFER_MLA,
# R1 dims + FP8 KV only; rejected by supports_combination
# otherwise. Behind FLASHINFER_MLA: wins past bs≈8, regresses
# at bs≤2.
AttentionBackendEnum.TOKENSPEED_MLA,
AttentionBackendEnum.CUTLASS_MLA,
AttentionBackendEnum.FLASH_ATTN_MLA,
AttentionBackendEnum.FLASHMLA,
@@ -43,6 +43,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
"vllm.v1.attention.backends.mla.prefill.trtllm_ragged."
"TrtllmRaggedPrefillBackend"
)
TOKENSPEED_MLA = (
"vllm.v1.attention.backends.mla.prefill.tokenspeed_mla."
"TokenspeedMLAPrefillBackend"
)
def get_path(self) -> str:
"""Get the fully qualified class path for this backend."""
@@ -67,6 +67,7 @@ def _get_mla_prefill_backend_priorities(
MLAPrefillBackendEnum.FLASH_ATTN,
MLAPrefillBackendEnum.TRTLLM_RAGGED,
MLAPrefillBackendEnum.FLASHINFER,
MLAPrefillBackendEnum.TOKENSPEED_MLA,
]
else: # Hopper (SM90) and older
return [
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TokenSpeed CuTe DSL backend for MLA prefill."""
from typing import TYPE_CHECKING
import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonPrefillMetadata,
)
from vllm.platforms.interface import DeviceCapability
class TokenspeedMLAPrefillBackend(MLAPrefillBackend):
"""TokenSpeed CuTe DSL backend for MLA prefill."""
requires_r1_mla_dimensions = True
@staticmethod
def get_name() -> str:
return "TOKENSPEED_MLA"
@classmethod
def supports_compute_capability(cls, device_capability: "DeviceCapability") -> bool:
return device_capability.major == 10
_INSTALL_HINT = (
"tokenspeed_mla package is not installed. "
"Install it with: `uv pip install tokenspeed-mla`"
)
@classmethod
def is_available(cls) -> bool:
try:
from tokenspeed_mla import (
tokenspeed_mla_prefill, # noqa: F401
)
return True
except ImportError:
return False
@classmethod
def validate_configuration(
cls,
device_capability,
selector_config,
) -> list[str]:
# Replace the generic "required dependencies not available" message
# from the base class with a specific install hint so users know
# exactly which package to install when they explicitly select this
# backend without having tokenspeed_mla installed.
reasons = super().validate_configuration(device_capability, selector_config)
return [
cls._INSTALL_HINT if r == "required dependencies not available" else r
for r in reasons
]
def __init__(
self,
num_heads: int,
scale: float,
kv_lora_rank: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
vllm_config: "VllmConfig",
) -> None:
super().__init__(
num_heads=num_heads,
scale=scale,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
# Pre-JIT BF16 and FP8 prefill kernels. Idempotent — also called from
# TokenspeedMLAImpl.__init__; second call is a no-op.
from tokenspeed_mla import warmup_compile_prefill
for q_dtype in (torch.bfloat16, torch.float8_e4m3fn):
warmup_compile_prefill(
q_dtype=q_dtype,
d_qk=qk_nope_head_dim + qk_rope_head_dim,
d_v=v_head_dim,
enable_pdl=False,
)
def prepare_metadata(
self,
prefill_metadata: "MLACommonPrefillMetadata",
) -> None:
super().prepare_metadata(prefill_metadata)
# Kernel signature requires `seq_lens` but the implementation never reads
# it (per-batch lengths are derived from `cum_seq_lens` diffs); compute
# for parity with trtllm_ragged. cuda-graph padding in
# `query_start_loc` is saturated to `total_num_tokens`
# (gpu_model_runner.py:1905), so trailing diffs are 0 and padded batches
# are kernel no-ops — same reason trtllm passes the padded length as
# batch_size directly.
self._query_seq_lens = (
prefill_metadata.query_start_loc[1:] - prefill_metadata.query_start_loc[:-1]
)
def run_prefill_new_tokens(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
return_softmax_lse: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
from tokenspeed_mla import tokenspeed_mla_prefill
# `v` arrives as the second half of `kv_nope.split(...)` in
# mla_attention.forward_mha — a non-contiguous view of `kv_nope` along
# dim=-1. The kernel does `v.reshape(1, total_kv, h_k, 1, d_v)` which
# would silently copy on a non-contiguous tensor; force contiguity here
# so the copy (if any) happens once outside the kernel call.
v = v.contiguous()
ret = tokenspeed_mla_prefill(
query=q,
key=k,
value=v,
seq_lens=self._query_seq_lens,
cum_seq_lens=self._prefill_metadata.query_start_loc,
max_seq_len=self._prefill_metadata.max_query_len,
batch_size=self._query_seq_lens.shape[0],
softmax_scale=self.scale,
is_causal=True,
return_lse=return_softmax_lse,
enable_pdl=False,
)
if isinstance(ret, tuple):
# Convert from (q_len, num_heads) to (num_heads, q_len)
return ret[0], ret[1].transpose(0, 1).contiguous()
return ret
def run_prefill_context_chunk(
self,
chunk_idx: int,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
from tokenspeed_mla import tokenspeed_mla_prefill
assert self._prefill_metadata.chunked_context is not None
chunked = self._prefill_metadata.chunked_context
# See note in run_prefill_new_tokens — `v` is a split-view of `kv_nope`
# in `_compute_prefill_context` and arrives non-contiguous.
v = v.contiguous()
attn_out, lse = tokenspeed_mla_prefill(
query=q,
key=k,
value=v,
seq_lens=chunked.seq_lens[chunk_idx],
cum_seq_lens=chunked.cu_seq_lens[chunk_idx],
max_seq_len=chunked.max_seq_lens[chunk_idx],
batch_size=chunked.seq_lens[chunk_idx].shape[0],
softmax_scale=self.scale,
is_causal=False,
return_lse=True,
cum_seq_lens_q=self._prefill_metadata.query_start_loc,
max_seq_len_q=self._prefill_metadata.max_query_len,
enable_pdl=False,
)
# Convert from (q_len, num_heads) to (num_heads, q_len)
return attn_out, lse.transpose(0, 1).contiguous()
@@ -0,0 +1,277 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TokenSpeed CuTe DSL MLA decode backend (Blackwell, FP8 KV cache only)."""
from typing import ClassVar
import torch
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonImpl,
MLACommonMetadata,
MLACommonMetadataBuilder,
QueryLenSupport,
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionLayer,
AttentionType,
MultipleOf,
)
from vllm.v1.attention.backends.utils import KVCacheLayoutType
logger = init_logger(__name__)
# Workspace upper bound for tokenspeed_mla_decode (per-device, lazy):
# num_sms * num_heads * MAX_Q_LEN * (kv_lora_rank + 1) * sizeof(float32)
# Matches the kernel's `get_workspace_size` formula. MAX_Q_LEN=8 covers up to
# EAGLE3 / MTP-2 spec decoding query lengths; larger q_len fails the kernel's
# own buffer check.
_TOKENSPEED_MAX_Q_LEN = 8
_g_workspace: dict[torch.device, torch.Tensor] = {}
def _get_workspace(
device: torch.device, num_heads: int, kv_lora_rank: int
) -> torch.Tensor:
from tokenspeed_mla import get_num_sm
needed = (
get_num_sm(device) * num_heads * _TOKENSPEED_MAX_Q_LEN * (kv_lora_rank + 1) * 4
)
existing = _g_workspace.get(device)
if existing is None or existing.numel() < needed:
_g_workspace[device] = torch.empty(needed, dtype=torch.int8, device=device)
return _g_workspace[device]
class TokenspeedMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
class TokenspeedMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"fp8",
"fp8_e4m3",
]
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [32, 64]
@staticmethod
def get_name() -> str:
return "TOKENSPEED_MLA"
@staticmethod
def get_impl_cls() -> type["TokenspeedMLAImpl"]:
return TokenspeedMLAImpl
@staticmethod
def get_builder_cls() -> type["TokenspeedMLAMetadataBuilder"]:
return TokenspeedMLAMetadataBuilder
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major == 10
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
device_capability: DeviceCapability,
) -> str | None:
# Surface a clear install hint up front rather than letting a raw
# ModuleNotFoundError fire deep inside `forward_mqa` at first request.
try:
import tokenspeed_mla # noqa: F401
except ImportError:
return (
"tokenspeed_mla package is not installed. "
"Install it with: `uv pip install tokenspeed-mla`"
)
# tokenspeed_mla CuTe DSL kernel is shape-specialized for DeepSeek R1
# MLA dimensions (qk_nope=128, qk_rope=64, v=128). Reject anything else.
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
if vllm_config.model_config is not None:
hf_text_config = vllm_config.model_config.hf_text_config
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 0)
qk_rope_head_dim = getattr(hf_text_config, "qk_rope_head_dim", 0)
v_head_dim = getattr(hf_text_config, "v_head_dim", 0)
if qk_nope_head_dim != 128 or qk_rope_head_dim != 64 or v_head_dim != 128:
return (
"tokenspeed_mla requires DeepSeek R1 MLA dimensions "
"(qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128), "
f"got ({qk_nope_head_dim}, {qk_rope_head_dim}, {v_head_dim})"
)
return None
@classmethod
def get_required_kv_cache_layout(cls) -> "KVCacheLayoutType | None":
return "HND"
class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]):
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
# MLA Specific Arguments
**mla_args,
) -> None:
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
**mla_args,
)
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
if any(unsupported_features):
raise NotImplementedError(
"TokenspeedMLAImpl does not support one of the following: "
"alibi_slopes, sliding_window, logits_soft_cap"
)
if attn_type != AttentionType.DECODER:
raise NotImplementedError(
"Encoder self-attention and "
"encoder/decoder cross-attention "
"are not implemented for "
"TokenspeedMLAImpl"
)
if not is_quantized_kv_cache(self.kv_cache_dtype):
raise NotImplementedError(
"TokenspeedMLAImpl requires an FP8 KV cache "
"(--kv-cache-dtype fp8 or fp8_e4m3); "
f"got kv_cache_dtype={self.kv_cache_dtype!r}."
)
# Allocate (or fetch the cached) workspace lazily on first forward —
# __init__ runs before the device is necessarily set on the worker;
# we know it for sure at forward time when we see the input tensor.
self._workspace_buffer: torch.Tensor | None = None
self.softmax_scale: float | None = None
self.output_scale: float | None = None
# Pre-JIT BF16 and FP8 prefill kernels here too — decode impl always
# runs when tokenspeed is selected, prefill backend may not (user can
# pair with flash_attn / trtllm). Idempotent.
from tokenspeed_mla import warmup_compile_prefill
for q_dtype in (torch.bfloat16, torch.float8_e4m3fn):
warmup_compile_prefill(
q_dtype=q_dtype,
d_qk=self.qk_nope_head_dim + self.qk_rope_head_dim,
d_v=self.v_head_dim,
enable_pdl=False,
)
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: MLACommonMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
from tokenspeed_mla import tokenspeed_mla_decode
assert kv_c_and_k_pe_cache.numel() > 0
assert attn_metadata.decode is not None
if isinstance(q, tuple):
q_nope, q_pe = q
q = torch.cat([q_nope, q_pe], dim=-1)
# supports_quant_query_input=True (set in MLACommonImpl) tells the
# pipeline to concat+FP8-quantize Q upstream via _decode_concat_quant_fp8_op.
# The kernel is shape-specialized for FP8 Q + FP8 KV, so anything else
# here means the upstream quant didn't run and the kernel will produce
# garbage.
assert q.dtype == torch.float8_e4m3fn, (
f"TokenspeedMLAImpl expected FP8 query (supports_quant_query_input=True), "
f"got {q.dtype}. Pipeline isinstance(q, tuple)={isinstance(q, tuple)}, "
f"q_scale={layer._q_scale_float}, k_scale={layer._k_scale_float}."
)
# tokenspeed_mla_decode expects query shape
# (num_decodes, q_len_per_request, num_heads, head_dim).
if attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
logger.warning_once(
"""TokenspeedMLAImpl got a query of uneven length.
This usually indicates an issue in batch reordering
or incorrect setup in dummy_run."""
)
q = q.unsqueeze(1)
else:
q = q.view(attn_metadata.num_decodes, -1, q.shape[-2], q.shape[-1])
if self.softmax_scale is None:
# FP8 KV cache is mandatory for this backend, so q_scale/k_scale
# always apply. softmax_scale is bmm1; output_scale is bmm2 — both
# required to recover the correct attention output from the FP8
# KV cache (V is stored as V_real/k_scale).
self.softmax_scale = (
self.scale * layer._q_scale_float * layer._k_scale_float
)
self.output_scale = layer._k_scale_float
if self._workspace_buffer is None:
self._workspace_buffer = _get_workspace(
q.device, self.num_heads, self.kv_lora_rank
)
# vLLM kv_c_and_k_pe_cache is already (num_blocks, block_size, head_size).
# tokenspeed_mla_decode wants 3D — pass as-is (no unsqueeze, unlike trtllm).
o = tokenspeed_mla_decode(
query=q,
kv_cache=kv_c_and_k_pe_cache,
workspace_buffer=self._workspace_buffer,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
block_tables=attn_metadata.decode.block_table,
seq_lens=attn_metadata.decode.seq_lens,
max_seq_len=attn_metadata.max_seq_len,
softmax_scale=self.softmax_scale,
output_scale=self.output_scale,
enable_pdl=False,
)
# Flatten the output for consistent shape
o = o.view(-1, o.shape[-2], o.shape[-1])
# tokenspeed_mla_decode does not return LSE.
return o, None
+3
View File
@@ -63,6 +63,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
FLASHINFER_MLA = (
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
)
TOKENSPEED_MLA = (
"vllm.v1.attention.backends.mla.tokenspeed_mla.TokenspeedMLABackend"
)
FLASHINFER_MLA_SPARSE = (
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse."
"FlashInferMLASparseBackend"