Add DCP + Eagle support for Tokenspeed MLA backends (#48180)

Signed-off-by: Pavani Majety <pmajety@nvidia.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Pavani Majety
2026-07-13 11:46:02 -07:00
committed by GitHub
co-authored by Jingyi Yang
parent 18c4067a54
commit 7fc97042c3
8 changed files with 313 additions and 23 deletions
@@ -154,6 +154,17 @@ def test_use_dcp_fallback(_mock):
assert _call(dcp_world_size=2) is False
@patch("vllm.utils.flashinfer.supports_trtllm_attention", return_value=True)
def test_use_dcp_fallback_prefill(_mock):
# TRTLLM prefill has no cross-rank LSE combine for DCP-sharded KV.
assert _call(dcp_world_size=2, is_prefill=True) is False
@patch("vllm.utils.flashinfer.supports_trtllm_attention", return_value=True)
def test_use_dcp_fallback_prefill_force_on_still_false(_mock):
assert _call(dcp_world_size=2, is_prefill=True, force_use_trtllm=True) is False
@patch("vllm.utils.flashinfer.supports_trtllm_attention", return_value=False)
def test_use_platform_unsupported(_mock):
assert _call() is False
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer GQA builder: reorder threshold under DCP with spec decode."""
import pytest
import torch
from tests.v1.attention.utils import create_vllm_config
from vllm.config import SpeculativeConfig, set_current_vllm_config
from vllm.platforms import current_platform
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.flashinfer import (
FlashInferDecodeKernel,
FlashInferMetadataBuilder,
)
from vllm.v1.attention.backends.utils import PerLayerParameters
from vllm.v1.kv_cache_interface import FullAttentionSpec
if not current_platform.is_cuda():
pytest.skip("FlashInfer backend requires a CUDA platform.", allow_module_level=True)
def test_flashinfer_gqa_dcp_spec_decode_clamps_reorder_threshold(monkeypatch):
"""trtllm-gen decode receives no cp_rank/global-seq-len information, so its
end-aligned causal mask is wrong for q_len > 1 over the DCP-interleaved
local KV shard. The builder must keep reorder_batch_threshold at 1 under
DCP so spec queries take the (DCP-aware) prefill path instead.
"""
vllm_config = create_vllm_config(max_model_len=1024)
vllm_config.parallel_config.decode_context_parallel_size = 2
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
monkeypatch.setattr(
flashinfer_backend, "can_use_trtllm_attention", lambda *args, **kwargs: True
)
monkeypatch.setattr(
FlashInferMetadataBuilder,
"_get_flashinfer_trtllm_api_decode_kernel",
staticmethod(lambda: FlashInferDecodeKernel.TRTLLM_GEN),
)
monkeypatch.setattr(
flashinfer_backend,
"get_per_layer_parameters",
lambda *args, **kwargs: {
"layer.0": PerLayerParameters(
window_left=-1, logits_soft_cap=None, sm_scale=0.1, has_sinks=False
)
},
)
kv_cache_spec = FullAttentionSpec(
block_size=16,
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
),
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
)
with set_current_vllm_config(vllm_config):
builder = FlashInferMetadataBuilder(
kv_cache_spec,
["layer.0"],
vllm_config,
torch.device("cpu"),
)
# Guard against passing vacuously with the kernel disabled.
assert (
builder.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
)
assert builder.reorder_batch_threshold == 1
+98
View File
@@ -8,6 +8,9 @@ Known Issues:
test_backend_correctness[small_prefill], but passes when run alone.
"""
import sys
from types import SimpleNamespace
import pytest
import torch
@@ -32,6 +35,7 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla import flashmla as flashmla_module
from vllm.v1.attention.backends.mla import tokenspeed_mla as tokenspeed_mla_module
from vllm.v1.attention.backends.mla.prefill import (
MLAPrefillBackendEnum,
get_mla_prefill_backend,
@@ -693,6 +697,100 @@ def test_mock_mla_dcp_fp8_decode_gathers_quantized_query(
)
def test_tokenspeed_mla_dcp_single_token_decode_contract(monkeypatch):
decode_call = None
num_decodes = 2
tokens_per_decode = 1
num_decode_tokens = num_decodes * tokens_per_decode
dcp_world_size = 2
dcp_rank = 1
num_heads = 128
kv_lora_rank = 512
qk_rope_head_dim = 64
head_size = kv_lora_rank + qk_rope_head_dim
block_size = 64
num_blocks = 4
max_seq_len = 24
def fake_decode(**kwargs):
nonlocal decode_call
decode_call = kwargs
q = kwargs["query"]
out = torch.empty(
q.shape[0],
q.shape[1],
q.shape[2],
kv_lora_rank,
dtype=torch.bfloat16,
)
lse = torch.empty(q.shape[0], q.shape[1], q.shape[2], dtype=torch.float32)
return out, lse
monkeypatch.setitem(
sys.modules,
"tokenspeed_mla",
SimpleNamespace(tokenspeed_mla_decode=fake_decode),
)
impl = object.__new__(tokenspeed_mla_module.TokenspeedMLAImpl)
impl.dcp_world_size = dcp_world_size
impl.dcp_rank = dcp_rank
impl.cp_kv_cache_interleave_size = 1
impl.need_to_return_lse_for_decode = True
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = qk_rope_head_dim
impl.num_heads = num_heads
impl.scale = 1.0
impl.softmax_scale = 1.0
impl.output_scale = 1.0
impl._workspace_buffer = torch.empty(1, dtype=torch.int8)
metadata = SimpleNamespace(
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
max_seq_len=max_seq_len,
decode=SimpleNamespace(
block_table=torch.empty((num_decodes, 1), dtype=torch.int32),
seq_lens=torch.tensor([16, max_seq_len], dtype=torch.int32),
dcp_tot_seq_lens=torch.tensor([16, max_seq_len], dtype=torch.int32),
),
)
q = torch.empty(num_decode_tokens, num_heads, head_size, dtype=torch.float8_e4m3fn)
kv_cache = torch.empty(
num_blocks,
block_size,
head_size,
dtype=torch.float8_e4m3fn,
)
out, lse = impl.forward_mqa(
q,
kv_cache,
metadata,
SimpleNamespace(_q_scale_float=2.0, _k_scale_float=3.0),
)
assert out.shape == (num_decode_tokens, num_heads, kv_lora_rank)
assert lse is not None
assert lse.shape == (num_decode_tokens, num_heads)
assert decode_call is not None
assert decode_call["query"].shape == (
num_decodes,
tokens_per_decode,
num_heads,
head_size,
)
torch.testing.assert_close(decode_call["seq_lens"], metadata.decode.seq_lens)
torch.testing.assert_close(decode_call["block_tables"], metadata.decode.block_table)
torch.testing.assert_close(
decode_call["causal_seqs"], metadata.decode.dcp_tot_seq_lens
)
assert decode_call["return_lse"] is True
assert decode_call["cp_world"] == dcp_world_size
assert decode_call["cp_rank"] == dcp_rank
@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"])
def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
monkeypatch, is_fp8_kvcache