[Bugfix][Attention] Restrict FlashInfer sparse MLA FULL graphs to decode (#59751)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -55,7 +55,6 @@ if not current_platform.is_cuda():
|
||||
)
|
||||
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
@@ -122,6 +121,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
qk_nope_head_dim: int = 128,
|
||||
v_head_dim: int = 128,
|
||||
stale_cpu_query_lens: list[int] | None = None,
|
||||
capture_graph: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Run a sparse-MLA backend with the given per-token indices and compute a
|
||||
dense per-token SDPA reference over the SAME indices.
|
||||
@@ -394,9 +394,25 @@ def _run_sparse_backend_vs_sdpa(
|
||||
metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device
|
||||
)
|
||||
with torch.inference_mode():
|
||||
impl.record_logical_topk_ready()
|
||||
backend_output = mock_layer.forward_impl(
|
||||
query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer
|
||||
)
|
||||
if capture_graph:
|
||||
device_boundaries = common_attn_metadata.query_start_loc.clone()
|
||||
common_attn_metadata.query_start_loc.copy_(
|
||||
common_attn_metadata.query_start_loc_cpu
|
||||
)
|
||||
metadata = builder.build(0, common_attn_metadata)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
impl.record_logical_topk_ready()
|
||||
backend_output = mock_layer.forward_impl(
|
||||
query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer
|
||||
)
|
||||
common_attn_metadata.query_start_loc.copy_(device_boundaries)
|
||||
builder.build(0, common_attn_metadata)
|
||||
graph.replay()
|
||||
return backend_output, sdpa_reference, causal_reference
|
||||
|
||||
|
||||
@@ -423,22 +439,19 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
|
||||
pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability")
|
||||
|
||||
|
||||
def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa(
|
||||
@pytest.mark.parametrize("query_lens", [[1, 7, 3, 5], [1, 7, 8, 0]])
|
||||
def test_flashinfer_sparse_mla_adaptive_varlen_graph_matches_sdpa(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
query_lens,
|
||||
):
|
||||
"""Adaptive request boundaries must drive SM100 sparse index conversion."""
|
||||
"""Replay uses device boundaries, including empty padding, despite stale CPU."""
|
||||
backend_cls = FlashInferMLASparseTRTLLMBackend
|
||||
_skip_if_backend_unavailable(backend_cls, "fp8", 64)
|
||||
assert (
|
||||
backend_cls.get_builder_cls().get_cudagraph_support(None, None)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
seq_lens = [257, 270, 265, 276]
|
||||
query_lens = [1, 7, 3, 5]
|
||||
sparse_indices = _build_dspark_noncausal_indices(
|
||||
seq_lens,
|
||||
query_lens,
|
||||
@@ -459,6 +472,7 @@ def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa(
|
||||
qk_nope_head_dim=192,
|
||||
v_head_dim=256,
|
||||
stale_cpu_query_lens=[4, 4, 4, 4],
|
||||
capture_graph=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
|
||||
@@ -16,7 +16,7 @@ import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
|
||||
@@ -228,6 +228,47 @@ def test_varlen_cudagraph_unsupported_backend_checks_scoped_bounds():
|
||||
assert unsupported([[target, never]], config, 1) == ("_DraftBackend", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [0, 7])
|
||||
def test_flashinfer_sparse_full_graphs_exclude_prefill_keep_varlen_decode(
|
||||
num_speculative_tokens,
|
||||
):
|
||||
"""Prefill must fall back without disabling adaptive verification graphs."""
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
|
||||
backend = FlashInferMLASparseTRTLLMBackend
|
||||
config: Any = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens, parallel_drafting=False
|
||||
),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=64, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
|
||||
)
|
||||
group = AttentionGroup(backend, ["target"], spec, 0)
|
||||
group.metadata_builders = [object.__new__(backend.get_builder_cls())]
|
||||
groups = [[group]]
|
||||
support = get_attn_cg_support(groups, config)
|
||||
compilation = CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL)
|
||||
mode = compilation.resolve_cudagraph_mode_and_sizes(
|
||||
support.min_cg_support,
|
||||
support.min_cg_attn_backend,
|
||||
uniform_decode_query_len=1 + num_speculative_tokens,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
assert mode.mixed_mode() != CUDAGraphMode.FULL
|
||||
assert mode.decode_mode() == CUDAGraphMode.FULL
|
||||
assert get_query_lens_mismatch_unsupported_backend(groups) is None
|
||||
unsupported = attn_utils.get_varlen_cudagraph_unsupported_backend
|
||||
assert unsupported(groups, config, 1 + num_speculative_tokens) is None
|
||||
assert unsupported(groups, config, 2 + num_speculative_tokens) == (
|
||||
backend.__name__,
|
||||
1 + num_speculative_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Fast prefill applies to the contiguous suffix of KV-sharing layers.
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.attention.backend import (
|
||||
AttentionType,
|
||||
MLAAttentionImpl,
|
||||
MultipleOf,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
@@ -34,7 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_convert_req_index_to_global_index,
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.deepseek_v2 import Indexer
|
||||
@@ -272,7 +273,16 @@ class FlashInferMLASparseMetadataBuilder(
|
||||
class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilder):
|
||||
"""Metadata builder for the SM100 TRT-LLM sparse MLA kernel."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
# Decode uses device request boundaries; prefill metadata is not graph-safe.
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user