[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell (#41778)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -1362,6 +1362,7 @@ def backend_supports_prefill_query_quantization() -> bool:
|
||||
return backend_cls.get_name() in (
|
||||
"FLASHINFER",
|
||||
"TRTLLM_RAGGED",
|
||||
"TOKENSPEED_MLA",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -110,6 +110,10 @@ def _get_backend_priorities(
|
||||
|
||||
return [
|
||||
AttentionBackendEnum.FLASHINFER_MLA,
|
||||
# R1 dims + FP8 KV only; rejected by supports_combination
|
||||
# otherwise. Behind FLASHINFER_MLA: wins past bs≈8, regresses
|
||||
# at bs≤2.
|
||||
AttentionBackendEnum.TOKENSPEED_MLA,
|
||||
AttentionBackendEnum.CUTLASS_MLA,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA,
|
||||
AttentionBackendEnum.FLASHMLA,
|
||||
|
||||
@@ -43,6 +43,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.mla.prefill.trtllm_ragged."
|
||||
"TrtllmRaggedPrefillBackend"
|
||||
)
|
||||
TOKENSPEED_MLA = (
|
||||
"vllm.v1.attention.backends.mla.prefill.tokenspeed_mla."
|
||||
"TokenspeedMLAPrefillBackend"
|
||||
)
|
||||
|
||||
def get_path(self) -> str:
|
||||
"""Get the fully qualified class path for this backend."""
|
||||
|
||||
@@ -67,6 +67,7 @@ def _get_mla_prefill_backend_priorities(
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
MLAPrefillBackendEnum.TRTLLM_RAGGED,
|
||||
MLAPrefillBackendEnum.FLASHINFER,
|
||||
MLAPrefillBackendEnum.TOKENSPEED_MLA,
|
||||
]
|
||||
else: # Hopper (SM90) and older
|
||||
return [
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TokenSpeed CuTe DSL backend for MLA prefill."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonPrefillMetadata,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
|
||||
class TokenspeedMLAPrefillBackend(MLAPrefillBackend):
|
||||
"""TokenSpeed CuTe DSL backend for MLA prefill."""
|
||||
|
||||
requires_r1_mla_dimensions = True
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TOKENSPEED_MLA"
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, device_capability: "DeviceCapability") -> bool:
|
||||
return device_capability.major == 10
|
||||
|
||||
_INSTALL_HINT = (
|
||||
"tokenspeed_mla package is not installed. "
|
||||
"Install it with: `uv pip install tokenspeed-mla`"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
try:
|
||||
from tokenspeed_mla import (
|
||||
tokenspeed_mla_prefill, # noqa: F401
|
||||
)
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def validate_configuration(
|
||||
cls,
|
||||
device_capability,
|
||||
selector_config,
|
||||
) -> list[str]:
|
||||
# Replace the generic "required dependencies not available" message
|
||||
# from the base class with a specific install hint so users know
|
||||
# exactly which package to install when they explicitly select this
|
||||
# backend without having tokenspeed_mla installed.
|
||||
reasons = super().validate_configuration(device_capability, selector_config)
|
||||
return [
|
||||
cls._INSTALL_HINT if r == "required dependencies not available" else r
|
||||
for r in reasons
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
scale: float,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads=num_heads,
|
||||
scale=scale,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
# Pre-JIT BF16 and FP8 prefill kernels. Idempotent — also called from
|
||||
# TokenspeedMLAImpl.__init__; second call is a no-op.
|
||||
from tokenspeed_mla import warmup_compile_prefill
|
||||
|
||||
for q_dtype in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
warmup_compile_prefill(
|
||||
q_dtype=q_dtype,
|
||||
d_qk=qk_nope_head_dim + qk_rope_head_dim,
|
||||
d_v=v_head_dim,
|
||||
enable_pdl=False,
|
||||
)
|
||||
|
||||
def prepare_metadata(
|
||||
self,
|
||||
prefill_metadata: "MLACommonPrefillMetadata",
|
||||
) -> None:
|
||||
super().prepare_metadata(prefill_metadata)
|
||||
# Kernel signature requires `seq_lens` but the implementation never reads
|
||||
# it (per-batch lengths are derived from `cum_seq_lens` diffs); compute
|
||||
# for parity with trtllm_ragged. cuda-graph padding in
|
||||
# `query_start_loc` is saturated to `total_num_tokens`
|
||||
# (gpu_model_runner.py:1905), so trailing diffs are 0 and padded batches
|
||||
# are kernel no-ops — same reason trtllm passes the padded length as
|
||||
# batch_size directly.
|
||||
self._query_seq_lens = (
|
||||
prefill_metadata.query_start_loc[1:] - prefill_metadata.query_start_loc[:-1]
|
||||
)
|
||||
|
||||
def run_prefill_new_tokens(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
return_softmax_lse: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
from tokenspeed_mla import tokenspeed_mla_prefill
|
||||
|
||||
# `v` arrives as the second half of `kv_nope.split(...)` in
|
||||
# mla_attention.forward_mha — a non-contiguous view of `kv_nope` along
|
||||
# dim=-1. The kernel does `v.reshape(1, total_kv, h_k, 1, d_v)` which
|
||||
# would silently copy on a non-contiguous tensor; force contiguity here
|
||||
# so the copy (if any) happens once outside the kernel call.
|
||||
v = v.contiguous()
|
||||
|
||||
ret = tokenspeed_mla_prefill(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
seq_lens=self._query_seq_lens,
|
||||
cum_seq_lens=self._prefill_metadata.query_start_loc,
|
||||
max_seq_len=self._prefill_metadata.max_query_len,
|
||||
batch_size=self._query_seq_lens.shape[0],
|
||||
softmax_scale=self.scale,
|
||||
is_causal=True,
|
||||
return_lse=return_softmax_lse,
|
||||
enable_pdl=False,
|
||||
)
|
||||
|
||||
if isinstance(ret, tuple):
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return ret[0], ret[1].transpose(0, 1).contiguous()
|
||||
return ret
|
||||
|
||||
def run_prefill_context_chunk(
|
||||
self,
|
||||
chunk_idx: int,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from tokenspeed_mla import tokenspeed_mla_prefill
|
||||
|
||||
assert self._prefill_metadata.chunked_context is not None
|
||||
chunked = self._prefill_metadata.chunked_context
|
||||
|
||||
# See note in run_prefill_new_tokens — `v` is a split-view of `kv_nope`
|
||||
# in `_compute_prefill_context` and arrives non-contiguous.
|
||||
v = v.contiguous()
|
||||
|
||||
attn_out, lse = tokenspeed_mla_prefill(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
seq_lens=chunked.seq_lens[chunk_idx],
|
||||
cum_seq_lens=chunked.cu_seq_lens[chunk_idx],
|
||||
max_seq_len=chunked.max_seq_lens[chunk_idx],
|
||||
batch_size=chunked.seq_lens[chunk_idx].shape[0],
|
||||
softmax_scale=self.scale,
|
||||
is_causal=False,
|
||||
return_lse=True,
|
||||
cum_seq_lens_q=self._prefill_metadata.query_start_loc,
|
||||
max_seq_len_q=self._prefill_metadata.max_query_len,
|
||||
enable_pdl=False,
|
||||
)
|
||||
|
||||
# Convert from (q_len, num_heads) to (num_heads, q_len)
|
||||
return attn_out, lse.transpose(0, 1).contiguous()
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TokenSpeed CuTe DSL MLA decode backend (Blackwell, FP8 KV cache only)."""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
QueryLenSupport,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionLayer,
|
||||
AttentionType,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Workspace upper bound for tokenspeed_mla_decode (per-device, lazy):
|
||||
# num_sms * num_heads * MAX_Q_LEN * (kv_lora_rank + 1) * sizeof(float32)
|
||||
# Matches the kernel's `get_workspace_size` formula. MAX_Q_LEN=8 covers up to
|
||||
# EAGLE3 / MTP-2 spec decoding query lengths; larger q_len fails the kernel's
|
||||
# own buffer check.
|
||||
_TOKENSPEED_MAX_Q_LEN = 8
|
||||
|
||||
_g_workspace: dict[torch.device, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _get_workspace(
|
||||
device: torch.device, num_heads: int, kv_lora_rank: int
|
||||
) -> torch.Tensor:
|
||||
from tokenspeed_mla import get_num_sm
|
||||
|
||||
needed = (
|
||||
get_num_sm(device) * num_heads * _TOKENSPEED_MAX_Q_LEN * (kv_lora_rank + 1) * 4
|
||||
)
|
||||
existing = _g_workspace.get(device)
|
||||
if existing is None or existing.numel() < needed:
|
||||
_g_workspace[device] = torch.empty(needed, dtype=torch.int8, device=device)
|
||||
return _g_workspace[device]
|
||||
|
||||
|
||||
class TokenspeedMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
|
||||
|
||||
|
||||
class TokenspeedMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [32, 64]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TOKENSPEED_MLA"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["TokenspeedMLAImpl"]:
|
||||
return TokenspeedMLAImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["TokenspeedMLAMetadataBuilder"]:
|
||||
return TokenspeedMLAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major == 10
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# Surface a clear install hint up front rather than letting a raw
|
||||
# ModuleNotFoundError fire deep inside `forward_mqa` at first request.
|
||||
try:
|
||||
import tokenspeed_mla # noqa: F401
|
||||
except ImportError:
|
||||
return (
|
||||
"tokenspeed_mla package is not installed. "
|
||||
"Install it with: `uv pip install tokenspeed-mla`"
|
||||
)
|
||||
|
||||
# tokenspeed_mla CuTe DSL kernel is shape-specialized for DeepSeek R1
|
||||
# MLA dimensions (qk_nope=128, qk_rope=64, v=128). Reject anything else.
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.model_config is not None:
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 0)
|
||||
qk_rope_head_dim = getattr(hf_text_config, "qk_rope_head_dim", 0)
|
||||
v_head_dim = getattr(hf_text_config, "v_head_dim", 0)
|
||||
if qk_nope_head_dim != 128 or qk_rope_head_dim != 64 or v_head_dim != 128:
|
||||
return (
|
||||
"tokenspeed_mla requires DeepSeek R1 MLA dimensions "
|
||||
"(qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128), "
|
||||
f"got ({qk_nope_head_dim}, {qk_rope_head_dim}, {v_head_dim})"
|
||||
)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_required_kv_cache_layout(cls) -> "KVCacheLayoutType | None":
|
||||
return "HND"
|
||||
|
||||
|
||||
class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
**mla_args,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
alibi_slopes,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
logits_soft_cap,
|
||||
attn_type,
|
||||
kv_sharing_target_layer_name,
|
||||
**mla_args,
|
||||
)
|
||||
|
||||
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
|
||||
if any(unsupported_features):
|
||||
raise NotImplementedError(
|
||||
"TokenspeedMLAImpl does not support one of the following: "
|
||||
"alibi_slopes, sliding_window, logits_soft_cap"
|
||||
)
|
||||
|
||||
if attn_type != AttentionType.DECODER:
|
||||
raise NotImplementedError(
|
||||
"Encoder self-attention and "
|
||||
"encoder/decoder cross-attention "
|
||||
"are not implemented for "
|
||||
"TokenspeedMLAImpl"
|
||||
)
|
||||
|
||||
if not is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
raise NotImplementedError(
|
||||
"TokenspeedMLAImpl requires an FP8 KV cache "
|
||||
"(--kv-cache-dtype fp8 or fp8_e4m3); "
|
||||
f"got kv_cache_dtype={self.kv_cache_dtype!r}."
|
||||
)
|
||||
|
||||
# Allocate (or fetch the cached) workspace lazily on first forward —
|
||||
# __init__ runs before the device is necessarily set on the worker;
|
||||
# we know it for sure at forward time when we see the input tensor.
|
||||
self._workspace_buffer: torch.Tensor | None = None
|
||||
self.softmax_scale: float | None = None
|
||||
self.output_scale: float | None = None
|
||||
|
||||
# Pre-JIT BF16 and FP8 prefill kernels here too — decode impl always
|
||||
# runs when tokenspeed is selected, prefill backend may not (user can
|
||||
# pair with flash_attn / trtllm). Idempotent.
|
||||
from tokenspeed_mla import warmup_compile_prefill
|
||||
|
||||
for q_dtype in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
warmup_compile_prefill(
|
||||
q_dtype=q_dtype,
|
||||
d_qk=self.qk_nope_head_dim + self.qk_rope_head_dim,
|
||||
d_v=self.v_head_dim,
|
||||
enable_pdl=False,
|
||||
)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
from tokenspeed_mla import tokenspeed_mla_decode
|
||||
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
assert attn_metadata.decode is not None
|
||||
|
||||
if isinstance(q, tuple):
|
||||
q_nope, q_pe = q
|
||||
q = torch.cat([q_nope, q_pe], dim=-1)
|
||||
|
||||
# supports_quant_query_input=True (set in MLACommonImpl) tells the
|
||||
# pipeline to concat+FP8-quantize Q upstream via _decode_concat_quant_fp8_op.
|
||||
# The kernel is shape-specialized for FP8 Q + FP8 KV, so anything else
|
||||
# here means the upstream quant didn't run and the kernel will produce
|
||||
# garbage.
|
||||
assert q.dtype == torch.float8_e4m3fn, (
|
||||
f"TokenspeedMLAImpl expected FP8 query (supports_quant_query_input=True), "
|
||||
f"got {q.dtype}. Pipeline isinstance(q, tuple)={isinstance(q, tuple)}, "
|
||||
f"q_scale={layer._q_scale_float}, k_scale={layer._k_scale_float}."
|
||||
)
|
||||
|
||||
# tokenspeed_mla_decode expects query shape
|
||||
# (num_decodes, q_len_per_request, num_heads, head_dim).
|
||||
if attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
logger.warning_once(
|
||||
"""TokenspeedMLAImpl got a query of uneven length.
|
||||
This usually indicates an issue in batch reordering
|
||||
or incorrect setup in dummy_run."""
|
||||
)
|
||||
q = q.unsqueeze(1)
|
||||
else:
|
||||
q = q.view(attn_metadata.num_decodes, -1, q.shape[-2], q.shape[-1])
|
||||
|
||||
if self.softmax_scale is None:
|
||||
# FP8 KV cache is mandatory for this backend, so q_scale/k_scale
|
||||
# always apply. softmax_scale is bmm1; output_scale is bmm2 — both
|
||||
# required to recover the correct attention output from the FP8
|
||||
# KV cache (V is stored as V_real/k_scale).
|
||||
self.softmax_scale = (
|
||||
self.scale * layer._q_scale_float * layer._k_scale_float
|
||||
)
|
||||
self.output_scale = layer._k_scale_float
|
||||
|
||||
if self._workspace_buffer is None:
|
||||
self._workspace_buffer = _get_workspace(
|
||||
q.device, self.num_heads, self.kv_lora_rank
|
||||
)
|
||||
|
||||
# vLLM kv_c_and_k_pe_cache is already (num_blocks, block_size, head_size).
|
||||
# tokenspeed_mla_decode wants 3D — pass as-is (no unsqueeze, unlike trtllm).
|
||||
o = tokenspeed_mla_decode(
|
||||
query=q,
|
||||
kv_cache=kv_c_and_k_pe_cache,
|
||||
workspace_buffer=self._workspace_buffer,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=attn_metadata.decode.block_table,
|
||||
seq_lens=attn_metadata.decode.seq_lens,
|
||||
max_seq_len=attn_metadata.max_seq_len,
|
||||
softmax_scale=self.softmax_scale,
|
||||
output_scale=self.output_scale,
|
||||
enable_pdl=False,
|
||||
)
|
||||
|
||||
# Flatten the output for consistent shape
|
||||
o = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
|
||||
# tokenspeed_mla_decode does not return LSE.
|
||||
return o, None
|
||||
@@ -63,6 +63,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
FLASHINFER_MLA = (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
|
||||
)
|
||||
TOKENSPEED_MLA = (
|
||||
"vllm.v1.attention.backends.mla.tokenspeed_mla.TokenspeedMLABackend"
|
||||
)
|
||||
FLASHINFER_MLA_SPARSE = (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse."
|
||||
"FlashInferMLASparseBackend"
|
||||
|
||||
Reference in New Issue
Block a user