[Bugfix][Attention] Guard sparse MLA masked MHA workspace (#50906)

Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
yimdev
2026-08-04 09:59:33 -04:00
committed by GitHub
co-authored by OpenAI Codex Matthew Bonanni
parent 7c40d61eaf
commit 199644d410
3 changed files with 115 additions and 13 deletions
@@ -21,6 +21,11 @@ from tests.v1.attention.utils import (
)
from vllm import _custom_ops as ops
from vllm.config import set_current_vllm_config
from vllm.model_executor.layers.attention.sparse_mla_attention import (
GLOBAL_TOPK_MASK_MAX_BYTES,
_masked_mha_workspace_fits,
_topk_mask_shape,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.platforms import current_platform
@@ -859,6 +864,38 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
assert out == expected
@pytest.mark.parametrize(
("max_query_len", "expected"),
[(32768, True), (33024, False)],
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
"""A 32K prefill needs the default workspace exactly; shrinking it would
push a supported request onto MQA."""
assert (
_masked_mha_workspace_fits(
batch_size=1,
max_query_len=max_query_len,
context_chunk_max_seq_lens=None,
workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
)
is expected
)
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
"""Request count and context chunk length are independent multipliers."""
base = dict(batch_size=2, max_query_len=2048, context_chunk_max_seq_lens=[2048])
exact = math.prod(_topk_mask_shape(2, 2048, 2048))
assert _masked_mha_workspace_fits(**base, workspace_numel=exact)
assert not _masked_mha_workspace_fits(
**{**base, "batch_size": 3}, workspace_numel=exact
)
assert not _masked_mha_workspace_fits(
**{**base, "context_chunk_max_seq_lens": [4096]}, workspace_numel=exact
)
PREFILL_BATCH_SPECS = {
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
@@ -769,6 +769,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
query_len=prefill.max_query_len,
seq_len=prefill_max_seq_len,
)
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
)
use_mha = (use_dense_mha or use_masked_mha) and not (
self._vllm_config.attention_config.sparse_mla_force_mqa
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
import math
from shutil import which
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
@@ -40,7 +41,38 @@ logger = init_logger(__name__)
T = TypeVar("T", bound=AttentionMetadata)
GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB
def _topk_mask_shape(
batch_size: int,
max_query_len: int,
max_key_len: int,
reserve_key_starts_word: bool = False,
) -> tuple[int, int, int]:
"""Shape of a bit-packed top-k mask, shared by every site that builds one."""
tile_m = 128 if max_query_len <= 128 else 256
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word)
return batch_size, padded_q_len, num_words
def _masked_mha_workspace_fits(
batch_size: int,
max_query_len: int,
context_chunk_max_seq_lens: list[int] | None,
workspace_numel: int,
) -> bool:
"""Return whether the suffix and per-context-chunk masks fit the workspace.
The global mask is excluded: it always needs more, and has its own check.
"""
max_key_len = max(
max_query_len,
max(context_chunk_max_seq_lens or (), default=0),
)
needed = math.prod(_topk_mask_shape(batch_size, max_query_len, max_key_len))
return needed <= workspace_numel
def _is_masked_mha_available(
@@ -504,6 +536,30 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
kv_cache_dtype=kv_cache_dtype,
)
@staticmethod
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
"""Whether this prefill batch's top-k masks fit the workspace."""
workspace = prefill.topk_mask_workspace
if workspace is None or prefill.query_lens_cpu is None:
return False
fits = _masked_mha_workspace_fits(
batch_size=len(prefill.query_lens_cpu),
max_query_len=prefill.max_query_len,
context_chunk_max_seq_lens=(
None
if prefill.chunked_context is None
else prefill.chunked_context.max_seq_lens
),
workspace_numel=workspace.numel(),
)
if not fits:
logger.warning_once(
"Sparse MLA top-k mask workspace (%d MiB) is too small for some "
"prefill batches; those fall back to slower sparse MQA.",
workspace.numel() * torch.int32.itemsize // (1024 * 1024),
)
return fits
@staticmethod
def _slice_topk_per_req(
topk_all: torch.Tensor,
@@ -554,12 +610,14 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
mask is too large, signalling the caller to fall back to per-chunk
index remapping.
"""
batch_size = len(q_lens)
tile_m = 128 if max_query_len <= 128 else 256
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
num_words_padded = (max_seq_len + 31) // 32 + 1
batch_size, padded_q_len, num_words_padded = _topk_mask_shape(
len(q_lens),
max_query_len,
max_seq_len,
reserve_key_starts_word=True,
)
needed = batch_size * padded_q_len * num_words_padded
if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES:
if needed > topk_mask_workspace.numel():
return None
mask = topk_mask_workspace[:needed].view(
@@ -597,15 +655,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
)
from vllm.vllm_flash_attn import flash_attn_varlen_func
tile_m = 128 if max_seqlen_q <= 128 else 256
padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m
if dense_mask is None:
batch_size = len(q_lens)
num_words = (max_seqlen_k + 31) // 32
assert topk_mask_workspace is not None
workspace_3d = topk_mask_workspace[
: batch_size * padded_q_len * num_words
].view(batch_size, padded_q_len, num_words)
batch_size, padded_q_len, num_words = _topk_mask_shape(
len(q_lens), max_seqlen_q, max_seqlen_k
)
words_needed = batch_size * padded_q_len * num_words
if words_needed > topk_mask_workspace.numel():
raise ValueError(
f"Sparse MLA top-k mask needs {words_needed} int32 words (batch="
f"{len(q_lens)}, q={max_seqlen_q}, k={max_seqlen_k}) but the "
f"workspace holds {topk_mask_workspace.numel()}."
)
workspace_3d = topk_mask_workspace[:words_needed].view(
batch_size, padded_q_len, num_words
)
dense_mask = _build_topk_mask(
topk_per_req,
q_lens,