[Bugfix][Attention] Restrict FlashInfer sparse MLA FULL graphs to decode (#59751)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-06 04:13:23 +00:00
committed by GitHub
parent a3e0243b1c
commit 30ab428235
3 changed files with 76 additions and 11 deletions
@@ -55,7 +55,6 @@ if not current_platform.is_cuda():
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
@@ -122,6 +121,7 @@ def _run_sparse_backend_vs_sdpa(
qk_nope_head_dim: int = 128,
v_head_dim: int = 128,
stale_cpu_query_lens: list[int] | None = None,
capture_graph: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Run a sparse-MLA backend with the given per-token indices and compute a
dense per-token SDPA reference over the SAME indices.
@@ -394,9 +394,25 @@ def _run_sparse_backend_vs_sdpa(
metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device
)
with torch.inference_mode():
impl.record_logical_topk_ready()
backend_output = mock_layer.forward_impl(
query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer
)
if capture_graph:
device_boundaries = common_attn_metadata.query_start_loc.clone()
common_attn_metadata.query_start_loc.copy_(
common_attn_metadata.query_start_loc_cpu
)
metadata = builder.build(0, common_attn_metadata)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
impl.record_logical_topk_ready()
backend_output = mock_layer.forward_impl(
query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer
)
common_attn_metadata.query_start_loc.copy_(device_boundaries)
builder.build(0, common_attn_metadata)
graph.replay()
return backend_output, sdpa_reference, causal_reference
@@ -423,22 +439,19 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability")
def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa(
@pytest.mark.parametrize("query_lens", [[1, 7, 3, 5], [1, 7, 8, 0]])
def test_flashinfer_sparse_mla_adaptive_varlen_graph_matches_sdpa(
default_vllm_config,
dist_init,
workspace_init,
query_lens,
):
"""Adaptive request boundaries must drive SM100 sparse index conversion."""
"""Replay uses device boundaries, including empty padding, despite stale CPU."""
backend_cls = FlashInferMLASparseTRTLLMBackend
_skip_if_backend_unavailable(backend_cls, "fp8", 64)
assert (
backend_cls.get_builder_cls().get_cudagraph_support(None, None)
== AttentionCGSupport.ALWAYS
)
device = torch.device(DEVICE_TYPE)
seq_lens = [257, 270, 265, 276]
query_lens = [1, 7, 3, 5]
sparse_indices = _build_dspark_noncausal_indices(
seq_lens,
query_lens,
@@ -459,6 +472,7 @@ def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa(
qk_nope_head_dim=192,
v_head_dim=256,
stale_cpu_query_lens=[4, 4, 4, 4],
capture_graph=True,
)
torch.testing.assert_close(
+42 -1
View File
@@ -16,7 +16,7 @@ import torch
import vllm.v1.hisparse.binding as attn_utils_module
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.config.compilation import CUDAGraphMode
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
@@ -228,6 +228,47 @@ def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
@pytest.mark.parametrize("num_speculative_tokens", [0, 7])
def test_flashinfer_sparse_full_graphs_exclude_prefill_keep_varlen_decode(
num_speculative_tokens,
):
"""Prefill must fall back without disabling adaptive verification graphs."""
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
backend = FlashInferMLASparseTRTLLMBackend
config: Any = SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=num_speculative_tokens, parallel_drafting=False
),
use_v2_model_runner=True,
)
spec = MLAAttentionSpec(
block_size=64, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
)
group = AttentionGroup(backend, ["target"], spec, 0)
group.metadata_builders = [object.__new__(backend.get_builder_cls())]
groups = [[group]]
support = get_attn_cg_support(groups, config)
compilation = CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL)
mode = compilation.resolve_cudagraph_mode_and_sizes(
support.min_cg_support,
support.min_cg_attn_backend,
uniform_decode_query_len=1 + num_speculative_tokens,
use_v2_model_runner=True,
)
assert mode.mixed_mode() != CUDAGraphMode.FULL
assert mode.decode_mode() == CUDAGraphMode.FULL
assert get_query_lens_mismatch_unsupported_backend(groups) is None
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
assert unsupported(groups, config, 1 + num_speculative_tokens) is None
assert unsupported(groups, config, 2 + num_speculative_tokens) == (
backend.__name__,
1 + num_speculative_tokens,
)
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.
@@ -26,6 +26,7 @@ from vllm.v1.attention.backend import (
AttentionType,
MLAAttentionImpl,
MultipleOf,
max_decode_query_len,
)
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.sparse_utils import (
@@ -34,7 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
triton_filter_and_convert_dcp_index,
)
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
if TYPE_CHECKING:
from vllm.model_executor.models.deepseek_v2 import Indexer
@@ -272,7 +273,16 @@ class FlashInferMLASparseMetadataBuilder(
class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilder):
"""Metadata builder for the SM100 TRT-LLM sparse MLA kernel."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
@classmethod
def get_varlen_cudagraph_max_query_len(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> int | None:
# Decode uses device request boundaries; prefill metadata is not graph-safe.
return max_decode_query_len(vllm_config)
def __init__(
self,