From 30ab428235e4a444dbb26f092d51c051480b8198 Mon Sep 17 00:00:00 2001 From: Tianyu Guo Date: Tue, 6 Oct 2026 12:13:23 +0800 Subject: [PATCH] [Bugfix][Attention] Restrict FlashInfer sparse MLA FULL graphs to decode (#59751) Signed-off-by: Tianyu Guo --- .../test_dspark_noncausal_sparse_mla.py | 30 +++++++++---- tests/v1/worker/test_attn_utils.py | 43 ++++++++++++++++++- .../backends/mla/flashinfer_mla_sparse.py | 14 +++++- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py index 3d4af17413..7cd7863b06 100644 --- a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py +++ b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py @@ -55,7 +55,6 @@ if not current_platform.is_cuda(): ) from vllm.utils.math_utils import cdiv -from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( FlashInferMLASparseTRTLLMBackend, ) @@ -122,6 +121,7 @@ def _run_sparse_backend_vs_sdpa( qk_nope_head_dim: int = 128, v_head_dim: int = 128, stale_cpu_query_lens: list[int] | None = None, + capture_graph: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Run a sparse-MLA backend with the given per-token indices and compute a dense per-token SDPA reference over the SAME indices. @@ -394,9 +394,25 @@ def _run_sparse_backend_vs_sdpa( metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device ) with torch.inference_mode(): + impl.record_logical_topk_ready() backend_output = mock_layer.forward_impl( query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer ) + if capture_graph: + device_boundaries = common_attn_metadata.query_start_loc.clone() + common_attn_metadata.query_start_loc.copy_( + common_attn_metadata.query_start_loc_cpu + ) + metadata = builder.build(0, common_attn_metadata) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + impl.record_logical_topk_ready() + backend_output = mock_layer.forward_impl( + query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer + ) + common_attn_metadata.query_start_loc.copy_(device_boundaries) + builder.build(0, common_attn_metadata) + graph.replay() return backend_output, sdpa_reference, causal_reference @@ -423,22 +439,19 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability") -def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa( +@pytest.mark.parametrize("query_lens", [[1, 7, 3, 5], [1, 7, 8, 0]]) +def test_flashinfer_sparse_mla_adaptive_varlen_graph_matches_sdpa( default_vllm_config, dist_init, workspace_init, + query_lens, ): - """Adaptive request boundaries must drive SM100 sparse index conversion.""" + """Replay uses device boundaries, including empty padding, despite stale CPU.""" backend_cls = FlashInferMLASparseTRTLLMBackend _skip_if_backend_unavailable(backend_cls, "fp8", 64) - assert ( - backend_cls.get_builder_cls().get_cudagraph_support(None, None) - == AttentionCGSupport.ALWAYS - ) device = torch.device(DEVICE_TYPE) seq_lens = [257, 270, 265, 276] - query_lens = [1, 7, 3, 5] sparse_indices = _build_dspark_noncausal_indices( seq_lens, query_lens, @@ -459,6 +472,7 @@ def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa( qk_nope_head_dim=192, v_head_dim=256, stale_cpu_query_lens=[4, 4, 4, 4], + capture_graph=True, ) torch.testing.assert_close( diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 0e6089d529..4114807052 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -16,7 +16,7 @@ import torch import vllm.v1.hisparse.binding as attn_utils_module from tests.v1.attention.utils import dense_kv_cache_views -from vllm.config.compilation import CUDAGraphMode +from vllm.config.compilation import CompilationConfig, CUDAGraphMode from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches @@ -228,6 +228,47 @@ def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds(): assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None) +@pytest.mark.parametrize("num_speculative_tokens", [0, 7]) +def test_flashinfer_sparse_full_graphs_exclude_prefill_keep_varlen_decode( + num_speculative_tokens, +): + """Prefill must fall back without disabling adaptive verification graphs.""" + from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( + FlashInferMLASparseTRTLLMBackend, + ) + + backend = FlashInferMLASparseTRTLLMBackend + config: Any = SimpleNamespace( + speculative_config=SimpleNamespace( + num_speculative_tokens=num_speculative_tokens, parallel_drafting=False + ), + use_v2_model_runner=True, + ) + spec = MLAAttentionSpec( + block_size=64, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 + ) + group = AttentionGroup(backend, ["target"], spec, 0) + group.metadata_builders = [object.__new__(backend.get_builder_cls())] + groups = [[group]] + support = get_attn_cg_support(groups, config) + compilation = CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL) + mode = compilation.resolve_cudagraph_mode_and_sizes( + support.min_cg_support, + support.min_cg_attn_backend, + uniform_decode_query_len=1 + num_speculative_tokens, + use_v2_model_runner=True, + ) + assert mode.mixed_mode() != CUDAGraphMode.FULL + assert mode.decode_mode() == CUDAGraphMode.FULL + assert get_query_lens_mismatch_unsupported_backend(groups) is None + unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend + assert unsupported(groups, config, 1 + num_speculative_tokens) is None + assert unsupported(groups, config, 2 + num_speculative_tokens) == ( + backend.__name__, + 1 + num_speculative_tokens, + ) + + def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch): """Fast prefill applies to the contiguous suffix of KV-sharing layers. diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 05af2de797..4ebda50178 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -26,6 +26,7 @@ from vllm.v1.attention.backend import ( AttentionType, MLAAttentionImpl, MultipleOf, + max_decode_query_len, ) from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup from vllm.v1.attention.backends.mla.sparse_utils import ( @@ -34,7 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, triton_filter_and_convert_dcp_index, ) -from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec if TYPE_CHECKING: from vllm.model_executor.models.deepseek_v2 import Indexer @@ -272,7 +273,16 @@ class FlashInferMLASparseMetadataBuilder( class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilder): """Metadata builder for the SM100 TRT-LLM sparse MLA kernel.""" - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + @classmethod + def get_varlen_cudagraph_max_query_len( + cls, + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> int | None: + # Decode uses device request boundaries; prefill metadata is not graph-safe. + return max_decode_query_len(vllm_config) def __init__( self,