[Cleanup][MLA] Remove FlashInfer DSpark DCP support (#53139)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Summer Yang
2026-08-21 11:42:52 -07:00
committed by GitHub
parent d9e0ace7a0
commit d6c2fec9fd
2 changed files with 56 additions and 184 deletions
+38 -63
View File
@@ -886,79 +886,54 @@ def test_tokenspeed_mla_noncausal_capability():
assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal()
def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch):
def test_flashinfer_mla_dspark_support_is_tp_only(monkeypatch):
flashinfer_mla_module = pytest.importorskip(
"vllm.v1.attention.backends.mla.flashinfer_mla"
)
decode_call = None
def fake_decode(**kwargs):
nonlocal decode_call
decode_call = kwargs
query = kwargs["query"]
output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16)
lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32)
return output, lse
monkeypatch.setattr(
flashinfer_mla_module,
"trtllm_batch_decode_with_kv_cache_mla",
fake_decode,
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(method="dspark"),
parallel_config=SimpleNamespace(decode_context_parallel_size=2),
model_config=None,
)
monkeypatch.setattr(
flashinfer_mla_module,
"_get_workspace_buffer",
lambda return_lse: torch.empty(1, dtype=torch.int8),
"get_current_vllm_config",
lambda: vllm_config,
)
impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl)
impl.dcp_world_size = 2
impl.dcp_rank = 1
impl.cp_kv_cache_interleave_size = 1
impl.need_to_return_lse_for_decode = True
impl.kv_lora_rank = 512
impl.qk_nope_head_dim = 128
impl.qk_rope_head_dim = 64
impl.bmm1_scale = 1.0
impl.bmm2_scale = 1.0
block_table = torch.tensor([[1], [2]], dtype=torch.int32)
metadata = SimpleNamespace(
num_decodes=2,
num_decode_tokens=6,
max_seq_len=7,
causal=True,
decode=SimpleNamespace(
block_table=block_table,
seq_lens=torch.tensor([5, 6], dtype=torch.int32),
dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32),
flattened_block_table=None,
flattened_seq_lens=None,
query_len=0,
),
)
query = torch.empty(6, 2, 576, dtype=torch.bfloat16)
kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16)
output, lse = impl.forward_mqa(
query,
kv_cache,
metadata,
SimpleNamespace(),
backend = flashinfer_mla_module.FlashInferMLABackend
builder = flashinfer_mla_module.FlashInferMLAMetadataBuilder
reason = backend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
device_capability=SimpleNamespace(),
)
assert output.shape == (6, 2, 512)
assert lse is not None
assert lse.shape == (6, 2)
assert decode_call is not None
assert decode_call["query"].shape == (6, 1, 2, 576)
torch.testing.assert_close(
decode_call["seq_lens"],
torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32),
)
torch.testing.assert_close(
decode_call["block_tables"], block_table.repeat_interleave(3, dim=0)
assert reason == "FlashInfer MLA does not support DSpark with DCP"
assert backend.supports_non_causal()
assert builder.supports_non_causal_multi_token_decode
assert not backend.supports_non_causal_dcp()
vllm_config.parallel_config.decode_context_parallel_size = 1
assert (
backend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
device_capability=SimpleNamespace(),
)
is None
)
+18 -121
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from typing import ClassVar
import torch
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
@@ -16,7 +15,6 @@ from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
MLACommonDecodeMetadata,
MLACommonImpl,
MLACommonMetadata,
MLACommonMetadataBuilder,
@@ -32,10 +30,6 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.utils import KVCacheLayoutType
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import AttentionSpec
logger = init_logger(__name__)
@@ -118,56 +112,12 @@ def _get_multi_ctas_kv_counter_buffer(
return _fi_multi_ctas_kv_counter
@dataclass
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
flattened_block_table: torch.Tensor | None = None
flattened_seq_lens: torch.Tensor | None = None
query_len: int = 0
@dataclass
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
pass
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
def __init__(
self,
kv_cache_spec: "AttentionSpec",
layer_names: list[str],
vllm_config: "VllmConfig",
device: torch.device,
) -> None:
super().__init__(
kv_cache_spec,
layer_names,
vllm_config,
device,
FlashInferMLAMetadata,
supports_dcp_with_varlen=True,
)
def _build_decode(
self,
block_table_tensor: torch.Tensor,
seq_lens_device: torch.Tensor,
max_seq_len: int,
query_start_loc_cpu: torch.Tensor,
query_start_loc_device: torch.Tensor,
num_decode_tokens: int,
dcp_tot_seq_lens_device: torch.Tensor | None,
) -> FlashInferMLADecodeMetadata:
return FlashInferMLADecodeMetadata(
block_table=block_table_tensor,
seq_lens=seq_lens_device,
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
)
class FlashInferMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
@@ -224,10 +174,16 @@ class FlashInferMLABackend(MLACommonBackend):
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
# FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192]
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
speculative_config = vllm_config.speculative_config
if (
speculative_config is not None
and speculative_config.method == "dspark"
and vllm_config.parallel_config.decode_context_parallel_size > 1
):
return "FlashInfer MLA does not support DSpark with DCP"
# FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192]
if vllm_config.model_config is not None:
hf_text_config = vllm_config.model_config.hf_text_config
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1)
@@ -243,7 +199,7 @@ class FlashInferMLABackend(MLACommonBackend):
return "HND"
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
can_return_lse_for_decode: bool = True
# trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own
# reference at flashinfer/trace/templates/attention.py:81:
@@ -314,7 +270,7 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: FlashInferMLAMetadata,
attn_metadata: MLACommonMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert kv_c_and_k_pe_cache.numel() > 0
@@ -326,28 +282,15 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
block_table = attn_metadata.decode.block_table
seq_lens = attn_metadata.decode.seq_lens
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
if not attn_metadata.causal:
# Non-causal DSpark block: flatten to single-token decode rows with
# per-row context seq_lens (trtllm-gen has no causal flag and would
# otherwise mask the block causally).
# FlashInfer decode has no causal flag. For TP-only DSpark, flatten
# each non-causal query block into independent single-token rows.
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
q = q.unsqueeze(1)
if query_len > 1:
block_table, seq_lens = self._prepare_flattened_decode_metadata(
attn_metadata,
query_len,
causal=False,
)
elif self.dcp_world_size > 1 and query_len > 1:
# Causal DCP block: flatten to single-token decode rows with
# per-row rank-local seq_lens for each query's visible prefix.
block_table, seq_lens = self._prepare_flattened_decode_metadata(
attn_metadata,
query_len,
causal=True,
)
q = q.unsqueeze(1)
block_table = block_table.repeat_interleave(query_len, dim=0)
seq_lens = seq_lens.repeat_interleave(query_len)
# trtllm API requires extra dimension q_len_per_request for MTP
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
logger.warning_once(
@@ -420,49 +363,3 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
o = o.view(-1, o.shape[-2], o.shape[-1])
return o, lse
def _prepare_flattened_decode_metadata(
self,
attn_metadata: FlashInferMLAMetadata,
query_len: int,
*,
causal: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Prepare flattened decode metadata once for all layers in the group."""
decode = attn_metadata.decode
assert decode is not None
if decode.query_len:
assert decode.query_len == query_len
assert decode.flattened_block_table is not None
assert decode.flattened_seq_lens is not None
return decode.flattened_block_table, decode.flattened_seq_lens
block_table = decode.block_table.repeat_interleave(query_len, dim=0)
if causal:
global_seq_lens = decode.dcp_tot_seq_lens
assert global_seq_lens is not None
offsets = torch.arange(
query_len - 1,
-1,
-1,
device=global_seq_lens.device,
dtype=global_seq_lens.dtype,
)
per_query_global_lens = torch.clamp(
(global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0
)
interleave = self.cp_kv_cache_interleave_size
dcp_span = self.dcp_world_size * interleave
remainder = torch.clamp(
per_query_global_lens % dcp_span - self.dcp_rank * interleave,
min=0,
max=interleave,
)
seq_lens = per_query_global_lens // dcp_span * interleave + remainder
else:
seq_lens = decode.seq_lens.repeat_interleave(query_len)
decode.flattened_block_table = block_table
decode.flattened_seq_lens = seq_lens
decode.query_len = query_len
return block_table, seq_lens