diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index cfce3c8bcf..e5c32aa490 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -886,79 +886,54 @@ def test_tokenspeed_mla_noncausal_capability(): assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal() -def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch): +def test_flashinfer_mla_dspark_support_is_tp_only(monkeypatch): flashinfer_mla_module = pytest.importorskip( "vllm.v1.attention.backends.mla.flashinfer_mla" ) - - decode_call = None - - def fake_decode(**kwargs): - nonlocal decode_call - decode_call = kwargs - query = kwargs["query"] - output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16) - lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32) - return output, lse - - monkeypatch.setattr( - flashinfer_mla_module, - "trtllm_batch_decode_with_kv_cache_mla", - fake_decode, + vllm_config = SimpleNamespace( + speculative_config=SimpleNamespace(method="dspark"), + parallel_config=SimpleNamespace(decode_context_parallel_size=2), + model_config=None, ) monkeypatch.setattr( flashinfer_mla_module, - "_get_workspace_buffer", - lambda return_lse: torch.empty(1, dtype=torch.int8), + "get_current_vllm_config", + lambda: vllm_config, ) - impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl) - impl.dcp_world_size = 2 - impl.dcp_rank = 1 - impl.cp_kv_cache_interleave_size = 1 - impl.need_to_return_lse_for_decode = True - impl.kv_lora_rank = 512 - impl.qk_nope_head_dim = 128 - impl.qk_rope_head_dim = 64 - impl.bmm1_scale = 1.0 - impl.bmm2_scale = 1.0 - - block_table = torch.tensor([[1], [2]], dtype=torch.int32) - metadata = SimpleNamespace( - num_decodes=2, - num_decode_tokens=6, - max_seq_len=7, - causal=True, - decode=SimpleNamespace( - block_table=block_table, - seq_lens=torch.tensor([5, 6], dtype=torch.int32), - dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32), - flattened_block_table=None, - flattened_seq_lens=None, - query_len=0, - ), - ) - query = torch.empty(6, 2, 576, dtype=torch.bfloat16) - kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16) - - output, lse = impl.forward_mqa( - query, - kv_cache, - metadata, - SimpleNamespace(), + backend = flashinfer_mla_module.FlashInferMLABackend + builder = flashinfer_mla_module.FlashInferMLAMetadataBuilder + reason = backend.supports_combination( + head_size=576, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + block_size=64, + use_mla=True, + has_sink=False, + use_sparse=False, + use_mm_prefix=False, + device_capability=SimpleNamespace(), ) - assert output.shape == (6, 2, 512) - assert lse is not None - assert lse.shape == (6, 2) - assert decode_call is not None - assert decode_call["query"].shape == (6, 1, 2, 576) - torch.testing.assert_close( - decode_call["seq_lens"], - torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32), - ) - torch.testing.assert_close( - decode_call["block_tables"], block_table.repeat_interleave(3, dim=0) + assert reason == "FlashInfer MLA does not support DSpark with DCP" + assert backend.supports_non_causal() + assert builder.supports_non_causal_multi_token_decode + assert not backend.supports_non_causal_dcp() + + vllm_config.parallel_config.decode_context_parallel_size = 1 + assert ( + backend.supports_combination( + head_size=576, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + block_size=64, + use_mla=True, + has_sink=False, + use_sparse=False, + use_mm_prefix=False, + device_capability=SimpleNamespace(), + ) + is None ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 79beb5332e..1712f06caf 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar +from typing import ClassVar import torch from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla @@ -16,7 +15,6 @@ from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( MLACommonBackend, - MLACommonDecodeMetadata, MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, @@ -32,10 +30,6 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.utils import KVCacheLayoutType -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.kv_cache_interface import AttentionSpec - logger = init_logger(__name__) @@ -118,56 +112,12 @@ def _get_multi_ctas_kv_counter_buffer( return _fi_multi_ctas_kv_counter -@dataclass -class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata): - flattened_block_table: torch.Tensor | None = None - flattened_seq_lens: torch.Tensor | None = None - query_len: int = 0 - - -@dataclass -class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]): - pass - - -class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]): +class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM # Non-causal DSpark blocks are flattened to single-token rows in forward_mqa. supports_non_causal_multi_token_decode: ClassVar[bool] = True - def __init__( - self, - kv_cache_spec: "AttentionSpec", - layer_names: list[str], - vllm_config: "VllmConfig", - device: torch.device, - ) -> None: - super().__init__( - kv_cache_spec, - layer_names, - vllm_config, - device, - FlashInferMLAMetadata, - supports_dcp_with_varlen=True, - ) - - def _build_decode( - self, - block_table_tensor: torch.Tensor, - seq_lens_device: torch.Tensor, - max_seq_len: int, - query_start_loc_cpu: torch.Tensor, - query_start_loc_device: torch.Tensor, - num_decode_tokens: int, - dcp_tot_seq_lens_device: torch.Tensor | None, - ) -> FlashInferMLADecodeMetadata: - return FlashInferMLADecodeMetadata( - block_table=block_table_tensor, - seq_lens=seq_lens_device, - dcp_tot_seq_lens=dcp_tot_seq_lens_device, - ) - class FlashInferMLABackend(MLACommonBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -224,10 +174,16 @@ class FlashInferMLABackend(MLACommonBackend): use_mm_prefix: bool, device_capability: DeviceCapability, ) -> str | None: - # FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192] - from vllm.config import get_current_vllm_config - vllm_config = get_current_vllm_config() + speculative_config = vllm_config.speculative_config + if ( + speculative_config is not None + and speculative_config.method == "dspark" + and vllm_config.parallel_config.decode_context_parallel_size > 1 + ): + return "FlashInfer MLA does not support DSpark with DCP" + + # FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192] if vllm_config.model_config is not None: hf_text_config = vllm_config.model_config.hf_text_config qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1) @@ -243,7 +199,7 @@ class FlashInferMLABackend(MLACommonBackend): return "HND" -class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): +class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): can_return_lse_for_decode: bool = True # trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own # reference at flashinfer/trace/templates/attention.py:81: @@ -314,7 +270,7 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], kv_c_and_k_pe_cache: torch.Tensor, - attn_metadata: FlashInferMLAMetadata, + attn_metadata: MLACommonMetadata, layer: AttentionLayer, ) -> tuple[torch.Tensor, torch.Tensor | None]: assert kv_c_and_k_pe_cache.numel() > 0 @@ -326,28 +282,15 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes if not attn_metadata.causal: - # Non-causal DSpark block: flatten to single-token decode rows with - # per-row context seq_lens (trtllm-gen has no causal flag and would - # otherwise mask the block causally). + # FlashInfer decode has no causal flag. For TP-only DSpark, flatten + # each non-causal query block into independent single-token rows. + query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes q = q.unsqueeze(1) if query_len > 1: - block_table, seq_lens = self._prepare_flattened_decode_metadata( - attn_metadata, - query_len, - causal=False, - ) - elif self.dcp_world_size > 1 and query_len > 1: - # Causal DCP block: flatten to single-token decode rows with - # per-row rank-local seq_lens for each query's visible prefix. - block_table, seq_lens = self._prepare_flattened_decode_metadata( - attn_metadata, - query_len, - causal=True, - ) - q = q.unsqueeze(1) + block_table = block_table.repeat_interleave(query_len, dim=0) + seq_lens = seq_lens.repeat_interleave(query_len) # trtllm API requires extra dimension q_len_per_request for MTP elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0: logger.warning_once( @@ -420,49 +363,3 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]): o = o.view(-1, o.shape[-2], o.shape[-1]) return o, lse - - def _prepare_flattened_decode_metadata( - self, - attn_metadata: FlashInferMLAMetadata, - query_len: int, - *, - causal: bool, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Prepare flattened decode metadata once for all layers in the group.""" - decode = attn_metadata.decode - assert decode is not None - if decode.query_len: - assert decode.query_len == query_len - assert decode.flattened_block_table is not None - assert decode.flattened_seq_lens is not None - return decode.flattened_block_table, decode.flattened_seq_lens - - block_table = decode.block_table.repeat_interleave(query_len, dim=0) - if causal: - global_seq_lens = decode.dcp_tot_seq_lens - assert global_seq_lens is not None - offsets = torch.arange( - query_len - 1, - -1, - -1, - device=global_seq_lens.device, - dtype=global_seq_lens.dtype, - ) - per_query_global_lens = torch.clamp( - (global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0 - ) - interleave = self.cp_kv_cache_interleave_size - dcp_span = self.dcp_world_size * interleave - remainder = torch.clamp( - per_query_global_lens % dcp_span - self.dcp_rank * interleave, - min=0, - max=interleave, - ) - seq_lens = per_query_global_lens // dcp_span * interleave + remainder - else: - seq_lens = decode.seq_lens.repeat_interleave(query_len) - - decode.flattened_block_table = block_table - decode.flattened_seq_lens = seq_lens - decode.query_len = query_len - return block_table, seq_lens