From 199644d410dbfd94255d6b0cde08479858032266 Mon Sep 17 00:00:00 2001 From: yimdev <5779256+yimdev@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:59:33 +0800 Subject: [PATCH] [Bugfix][Attention] Guard sparse MLA masked MHA workspace (#50906) Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com> Signed-off-by: Matthew Bonanni Co-authored-by: OpenAI Codex Co-authored-by: Matthew Bonanni --- .../v1/attention/test_sparse_mla_backends.py | 37 ++++++++ .../layers/attention/mla_attention.py | 1 + .../layers/attention/sparse_mla_attention.py | 90 ++++++++++++++++--- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 0c7b645c14..3d390d2afd 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -21,6 +21,11 @@ from tests.v1.attention.utils import ( ) from vllm import _custom_ops as ops from vllm.config import set_current_vllm_config +from vllm.model_executor.layers.attention.sparse_mla_attention import ( + GLOBAL_TOPK_MASK_MAX_BYTES, + _masked_mha_workspace_fits, + _topk_mask_shape, +) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.platforms import current_platform @@ -859,6 +864,38 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected): assert out == expected +@pytest.mark.parametrize( + ("max_query_len", "expected"), + [(32768, True), (33024, False)], +) +def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected): + """A 32K prefill needs the default workspace exactly; shrinking it would + push a supported request onto MQA.""" + assert ( + _masked_mha_workspace_fits( + batch_size=1, + max_query_len=max_query_len, + context_chunk_max_seq_lens=None, + workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize, + ) + is expected + ) + + +def test_masked_mha_workspace_fits_accounts_for_batch_and_context(): + """Request count and context chunk length are independent multipliers.""" + base = dict(batch_size=2, max_query_len=2048, context_chunk_max_seq_lens=[2048]) + exact = math.prod(_topk_mask_shape(2, 2048, 2048)) + + assert _masked_mha_workspace_fits(**base, workspace_numel=exact) + assert not _masked_mha_workspace_fits( + **{**base, "batch_size": 3}, workspace_numel=exact + ) + assert not _masked_mha_workspace_fits( + **{**base, "context_chunk_max_seq_lens": [4096]}, workspace_numel=exact + ) + + PREFILL_BATCH_SPECS = { "short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]), "short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]), diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 53b62da912..239dbe4bef 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -769,6 +769,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): query_len=prefill.max_query_len, seq_len=prefill_max_seq_len, ) + and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined] ) use_mha = (use_dense_mha or use_masked_mha) and not ( self._vllm_config.attention_config.sparse_mla_force_mqa diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index 90934d2892..9fc112423b 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Shared MHA implementation and metadata builder for sparse MLA backends.""" +import math from shutil import which from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast @@ -40,7 +41,38 @@ logger = init_logger(__name__) T = TypeVar("T", bound=AttentionMetadata) -GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB +GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB + + +def _topk_mask_shape( + batch_size: int, + max_query_len: int, + max_key_len: int, + reserve_key_starts_word: bool = False, +) -> tuple[int, int, int]: + """Shape of a bit-packed top-k mask, shared by every site that builds one.""" + tile_m = 128 if max_query_len <= 128 else 256 + padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m + num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word) + return batch_size, padded_q_len, num_words + + +def _masked_mha_workspace_fits( + batch_size: int, + max_query_len: int, + context_chunk_max_seq_lens: list[int] | None, + workspace_numel: int, +) -> bool: + """Return whether the suffix and per-context-chunk masks fit the workspace. + + The global mask is excluded: it always needs more, and has its own check. + """ + max_key_len = max( + max_query_len, + max(context_chunk_max_seq_lens or (), default=0), + ) + needed = math.prod(_topk_mask_shape(batch_size, max_query_len, max_key_len)) + return needed <= workspace_numel def _is_masked_mha_available( @@ -504,6 +536,30 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): kv_cache_dtype=kv_cache_dtype, ) + @staticmethod + def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool: + """Whether this prefill batch's top-k masks fit the workspace.""" + workspace = prefill.topk_mask_workspace + if workspace is None or prefill.query_lens_cpu is None: + return False + fits = _masked_mha_workspace_fits( + batch_size=len(prefill.query_lens_cpu), + max_query_len=prefill.max_query_len, + context_chunk_max_seq_lens=( + None + if prefill.chunked_context is None + else prefill.chunked_context.max_seq_lens + ), + workspace_numel=workspace.numel(), + ) + if not fits: + logger.warning_once( + "Sparse MLA top-k mask workspace (%d MiB) is too small for some " + "prefill batches; those fall back to slower sparse MQA.", + workspace.numel() * torch.int32.itemsize // (1024 * 1024), + ) + return fits + @staticmethod def _slice_topk_per_req( topk_all: torch.Tensor, @@ -554,12 +610,14 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): mask is too large, signalling the caller to fall back to per-chunk index remapping. """ - batch_size = len(q_lens) - tile_m = 128 if max_query_len <= 128 else 256 - padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m - num_words_padded = (max_seq_len + 31) // 32 + 1 + batch_size, padded_q_len, num_words_padded = _topk_mask_shape( + len(q_lens), + max_query_len, + max_seq_len, + reserve_key_starts_word=True, + ) needed = batch_size * padded_q_len * num_words_padded - if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES: + if needed > topk_mask_workspace.numel(): return None mask = topk_mask_workspace[:needed].view( @@ -597,15 +655,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): ) from vllm.vllm_flash_attn import flash_attn_varlen_func - tile_m = 128 if max_seqlen_q <= 128 else 256 - padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m if dense_mask is None: - batch_size = len(q_lens) - num_words = (max_seqlen_k + 31) // 32 assert topk_mask_workspace is not None - workspace_3d = topk_mask_workspace[ - : batch_size * padded_q_len * num_words - ].view(batch_size, padded_q_len, num_words) + batch_size, padded_q_len, num_words = _topk_mask_shape( + len(q_lens), max_seqlen_q, max_seqlen_k + ) + words_needed = batch_size * padded_q_len * num_words + if words_needed > topk_mask_workspace.numel(): + raise ValueError( + f"Sparse MLA top-k mask needs {words_needed} int32 words (batch=" + f"{len(q_lens)}, q={max_seqlen_q}, k={max_seqlen_k}) but the " + f"workspace holds {topk_mask_workspace.numel()}." + ) + workspace_3d = topk_mask_workspace[:words_needed].view( + batch_size, padded_q_len, num_words + ) dense_mask = _build_topk_mask( topk_per_req, q_lens,