[Cleanup][MLA] Remove FlashInfer DSpark DCP support (#53139)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -886,79 +886,54 @@ def test_tokenspeed_mla_noncausal_capability():
|
||||
assert tokenspeed_mla_module.TokenspeedMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
def test_flashinfer_mla_dcp_multi_token_decode_uses_per_query_bounds(monkeypatch):
|
||||
def test_flashinfer_mla_dspark_support_is_tp_only(monkeypatch):
|
||||
flashinfer_mla_module = pytest.importorskip(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla"
|
||||
)
|
||||
|
||||
decode_call = None
|
||||
|
||||
def fake_decode(**kwargs):
|
||||
nonlocal decode_call
|
||||
decode_call = kwargs
|
||||
query = kwargs["query"]
|
||||
output = torch.empty(*query.shape[:-1], 512, dtype=torch.bfloat16)
|
||||
lse = torch.empty(query.shape[0], query.shape[-2], dtype=torch.float32)
|
||||
return output, lse
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mla_module,
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fake_decode,
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(method="dspark"),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=2),
|
||||
model_config=None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mla_module,
|
||||
"_get_workspace_buffer",
|
||||
lambda return_lse: torch.empty(1, dtype=torch.int8),
|
||||
"get_current_vllm_config",
|
||||
lambda: vllm_config,
|
||||
)
|
||||
|
||||
impl = object.__new__(flashinfer_mla_module.FlashInferMLAImpl)
|
||||
impl.dcp_world_size = 2
|
||||
impl.dcp_rank = 1
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
|
||||
block_table = torch.tensor([[1], [2]], dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
num_decodes=2,
|
||||
num_decode_tokens=6,
|
||||
max_seq_len=7,
|
||||
causal=True,
|
||||
decode=SimpleNamespace(
|
||||
block_table=block_table,
|
||||
seq_lens=torch.tensor([5, 6], dtype=torch.int32),
|
||||
dcp_tot_seq_lens=torch.tensor([10, 13], dtype=torch.int32),
|
||||
flattened_block_table=None,
|
||||
flattened_seq_lens=None,
|
||||
query_len=0,
|
||||
),
|
||||
)
|
||||
query = torch.empty(6, 2, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.empty(3, 16, 576, dtype=torch.bfloat16)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
backend = flashinfer_mla_module.FlashInferMLABackend
|
||||
builder = flashinfer_mla_module.FlashInferMLAMetadataBuilder
|
||||
reason = backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output.shape == (6, 2, 512)
|
||||
assert lse is not None
|
||||
assert lse.shape == (6, 2)
|
||||
assert decode_call is not None
|
||||
assert decode_call["query"].shape == (6, 1, 2, 576)
|
||||
torch.testing.assert_close(
|
||||
decode_call["seq_lens"],
|
||||
torch.tensor([4, 4, 5, 5, 6, 6], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
decode_call["block_tables"], block_table.repeat_interleave(3, dim=0)
|
||||
assert reason == "FlashInfer MLA does not support DSpark with DCP"
|
||||
assert backend.supports_non_causal()
|
||||
assert builder.supports_non_causal_multi_token_decode
|
||||
assert not backend.supports_non_causal_dcp()
|
||||
|
||||
vllm_config.parallel_config.decode_context_parallel_size = 1
|
||||
assert (
|
||||
backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(),
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
@@ -16,7 +15,6 @@ from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonDecodeMetadata,
|
||||
MLACommonImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonMetadataBuilder,
|
||||
@@ -32,10 +30,6 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -118,56 +112,12 @@ def _get_multi_ctas_kv_counter_buffer(
|
||||
return _fi_multi_ctas_kv_counter
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
flattened_block_table: torch.Tensor | None = None
|
||||
flattened_seq_lens: torch.Tensor | None = None
|
||||
query_len: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferMLAMetadata(MLACommonMetadata[FlashInferMLADecodeMetadata]):
|
||||
pass
|
||||
|
||||
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[FlashInferMLAMetadata]):
|
||||
class FlashInferMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
|
||||
# Non-causal DSpark blocks are flattened to single-token rows in forward_mqa.
|
||||
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: "AttentionSpec",
|
||||
layer_names: list[str],
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
kv_cache_spec,
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
FlashInferMLAMetadata,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
|
||||
def _build_decode(
|
||||
self,
|
||||
block_table_tensor: torch.Tensor,
|
||||
seq_lens_device: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
query_start_loc_cpu: torch.Tensor,
|
||||
query_start_loc_device: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
dcp_tot_seq_lens_device: torch.Tensor | None,
|
||||
) -> FlashInferMLADecodeMetadata:
|
||||
return FlashInferMLADecodeMetadata(
|
||||
block_table=block_table_tensor,
|
||||
seq_lens=seq_lens_device,
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
)
|
||||
|
||||
|
||||
class FlashInferMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
@@ -224,10 +174,16 @@ class FlashInferMLABackend(MLACommonBackend):
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192]
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.method == "dspark"
|
||||
and vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
return "FlashInfer MLA does not support DSpark with DCP"
|
||||
|
||||
# FlashInfer MLA kernel requires qk_nope_head_dim in [64, 128, 192]
|
||||
if vllm_config.model_config is not None:
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1)
|
||||
@@ -243,7 +199,7 @@ class FlashInferMLABackend(MLACommonBackend):
|
||||
return "HND"
|
||||
|
||||
|
||||
class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
# trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own
|
||||
# reference at flashinfer/trace/templates/attention.py:81:
|
||||
@@ -314,7 +270,7 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert kv_c_and_k_pe_cache.numel() > 0
|
||||
@@ -326,28 +282,15 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
|
||||
block_table = attn_metadata.decode.block_table
|
||||
seq_lens = attn_metadata.decode.seq_lens
|
||||
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
|
||||
|
||||
if not attn_metadata.causal:
|
||||
# Non-causal DSpark block: flatten to single-token decode rows with
|
||||
# per-row context seq_lens (trtllm-gen has no causal flag and would
|
||||
# otherwise mask the block causally).
|
||||
# FlashInfer decode has no causal flag. For TP-only DSpark, flatten
|
||||
# each non-causal query block into independent single-token rows.
|
||||
query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes
|
||||
q = q.unsqueeze(1)
|
||||
if query_len > 1:
|
||||
block_table, seq_lens = self._prepare_flattened_decode_metadata(
|
||||
attn_metadata,
|
||||
query_len,
|
||||
causal=False,
|
||||
)
|
||||
elif self.dcp_world_size > 1 and query_len > 1:
|
||||
# Causal DCP block: flatten to single-token decode rows with
|
||||
# per-row rank-local seq_lens for each query's visible prefix.
|
||||
block_table, seq_lens = self._prepare_flattened_decode_metadata(
|
||||
attn_metadata,
|
||||
query_len,
|
||||
causal=True,
|
||||
)
|
||||
q = q.unsqueeze(1)
|
||||
block_table = block_table.repeat_interleave(query_len, dim=0)
|
||||
seq_lens = seq_lens.repeat_interleave(query_len)
|
||||
# trtllm API requires extra dimension q_len_per_request for MTP
|
||||
elif attn_metadata.num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
logger.warning_once(
|
||||
@@ -420,49 +363,3 @@ class FlashInferMLAImpl(MLACommonImpl[FlashInferMLAMetadata]):
|
||||
o = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
|
||||
return o, lse
|
||||
|
||||
def _prepare_flattened_decode_metadata(
|
||||
self,
|
||||
attn_metadata: FlashInferMLAMetadata,
|
||||
query_len: int,
|
||||
*,
|
||||
causal: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepare flattened decode metadata once for all layers in the group."""
|
||||
decode = attn_metadata.decode
|
||||
assert decode is not None
|
||||
if decode.query_len:
|
||||
assert decode.query_len == query_len
|
||||
assert decode.flattened_block_table is not None
|
||||
assert decode.flattened_seq_lens is not None
|
||||
return decode.flattened_block_table, decode.flattened_seq_lens
|
||||
|
||||
block_table = decode.block_table.repeat_interleave(query_len, dim=0)
|
||||
if causal:
|
||||
global_seq_lens = decode.dcp_tot_seq_lens
|
||||
assert global_seq_lens is not None
|
||||
offsets = torch.arange(
|
||||
query_len - 1,
|
||||
-1,
|
||||
-1,
|
||||
device=global_seq_lens.device,
|
||||
dtype=global_seq_lens.dtype,
|
||||
)
|
||||
per_query_global_lens = torch.clamp(
|
||||
(global_seq_lens.unsqueeze(1) - offsets).reshape(-1), min=0
|
||||
)
|
||||
interleave = self.cp_kv_cache_interleave_size
|
||||
dcp_span = self.dcp_world_size * interleave
|
||||
remainder = torch.clamp(
|
||||
per_query_global_lens % dcp_span - self.dcp_rank * interleave,
|
||||
min=0,
|
||||
max=interleave,
|
||||
)
|
||||
seq_lens = per_query_global_lens // dcp_span * interleave + remainder
|
||||
else:
|
||||
seq_lens = decode.seq_lens.repeat_interleave(query_len)
|
||||
|
||||
decode.flattened_block_table = block_table
|
||||
decode.flattened_seq_lens = seq_lens
|
||||
decode.query_len = query_len
|
||||
return block_table, seq_lens
|
||||
|
||||
Reference in New Issue
Block a user