[Bugfix][Attention] Guard sparse MLA masked MHA workspace (#50906)
Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
OpenAI Codex
Matthew Bonanni
parent
7c40d61eaf
commit
199644d410
@@ -21,6 +21,11 @@ from tests.v1.attention.utils import (
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES,
|
||||
_masked_mha_workspace_fits,
|
||||
_topk_mask_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -859,6 +864,38 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
||||
assert out == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_query_len", "expected"),
|
||||
[(32768, True), (33024, False)],
|
||||
)
|
||||
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
|
||||
"""A 32K prefill needs the default workspace exactly; shrinking it would
|
||||
push a supported request onto MQA."""
|
||||
assert (
|
||||
_masked_mha_workspace_fits(
|
||||
batch_size=1,
|
||||
max_query_len=max_query_len,
|
||||
context_chunk_max_seq_lens=None,
|
||||
workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
|
||||
"""Request count and context chunk length are independent multipliers."""
|
||||
base = dict(batch_size=2, max_query_len=2048, context_chunk_max_seq_lens=[2048])
|
||||
exact = math.prod(_topk_mask_shape(2, 2048, 2048))
|
||||
|
||||
assert _masked_mha_workspace_fits(**base, workspace_numel=exact)
|
||||
assert not _masked_mha_workspace_fits(
|
||||
**{**base, "batch_size": 3}, workspace_numel=exact
|
||||
)
|
||||
assert not _masked_mha_workspace_fits(
|
||||
**{**base, "context_chunk_max_seq_lens": [4096]}, workspace_numel=exact
|
||||
)
|
||||
|
||||
|
||||
PREFILL_BATCH_SPECS = {
|
||||
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
|
||||
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
|
||||
|
||||
@@ -769,6 +769,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
query_len=prefill.max_query_len,
|
||||
seq_len=prefill_max_seq_len,
|
||||
)
|
||||
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
|
||||
)
|
||||
use_mha = (use_dense_mha or use_masked_mha) and not (
|
||||
self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
import math
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
@@ -40,7 +41,38 @@ logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB
|
||||
|
||||
|
||||
def _topk_mask_shape(
|
||||
batch_size: int,
|
||||
max_query_len: int,
|
||||
max_key_len: int,
|
||||
reserve_key_starts_word: bool = False,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Shape of a bit-packed top-k mask, shared by every site that builds one."""
|
||||
tile_m = 128 if max_query_len <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
|
||||
num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word)
|
||||
return batch_size, padded_q_len, num_words
|
||||
|
||||
|
||||
def _masked_mha_workspace_fits(
|
||||
batch_size: int,
|
||||
max_query_len: int,
|
||||
context_chunk_max_seq_lens: list[int] | None,
|
||||
workspace_numel: int,
|
||||
) -> bool:
|
||||
"""Return whether the suffix and per-context-chunk masks fit the workspace.
|
||||
|
||||
The global mask is excluded: it always needs more, and has its own check.
|
||||
"""
|
||||
max_key_len = max(
|
||||
max_query_len,
|
||||
max(context_chunk_max_seq_lens or (), default=0),
|
||||
)
|
||||
needed = math.prod(_topk_mask_shape(batch_size, max_query_len, max_key_len))
|
||||
return needed <= workspace_numel
|
||||
|
||||
|
||||
def _is_masked_mha_available(
|
||||
@@ -504,6 +536,30 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
|
||||
"""Whether this prefill batch's top-k masks fit the workspace."""
|
||||
workspace = prefill.topk_mask_workspace
|
||||
if workspace is None or prefill.query_lens_cpu is None:
|
||||
return False
|
||||
fits = _masked_mha_workspace_fits(
|
||||
batch_size=len(prefill.query_lens_cpu),
|
||||
max_query_len=prefill.max_query_len,
|
||||
context_chunk_max_seq_lens=(
|
||||
None
|
||||
if prefill.chunked_context is None
|
||||
else prefill.chunked_context.max_seq_lens
|
||||
),
|
||||
workspace_numel=workspace.numel(),
|
||||
)
|
||||
if not fits:
|
||||
logger.warning_once(
|
||||
"Sparse MLA top-k mask workspace (%d MiB) is too small for some "
|
||||
"prefill batches; those fall back to slower sparse MQA.",
|
||||
workspace.numel() * torch.int32.itemsize // (1024 * 1024),
|
||||
)
|
||||
return fits
|
||||
|
||||
@staticmethod
|
||||
def _slice_topk_per_req(
|
||||
topk_all: torch.Tensor,
|
||||
@@ -554,12 +610,14 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
mask is too large, signalling the caller to fall back to per-chunk
|
||||
index remapping.
|
||||
"""
|
||||
batch_size = len(q_lens)
|
||||
tile_m = 128 if max_query_len <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
|
||||
num_words_padded = (max_seq_len + 31) // 32 + 1
|
||||
batch_size, padded_q_len, num_words_padded = _topk_mask_shape(
|
||||
len(q_lens),
|
||||
max_query_len,
|
||||
max_seq_len,
|
||||
reserve_key_starts_word=True,
|
||||
)
|
||||
needed = batch_size * padded_q_len * num_words_padded
|
||||
if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES:
|
||||
if needed > topk_mask_workspace.numel():
|
||||
return None
|
||||
|
||||
mask = topk_mask_workspace[:needed].view(
|
||||
@@ -597,15 +655,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
)
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
|
||||
tile_m = 128 if max_seqlen_q <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m
|
||||
if dense_mask is None:
|
||||
batch_size = len(q_lens)
|
||||
num_words = (max_seqlen_k + 31) // 32
|
||||
assert topk_mask_workspace is not None
|
||||
workspace_3d = topk_mask_workspace[
|
||||
: batch_size * padded_q_len * num_words
|
||||
].view(batch_size, padded_q_len, num_words)
|
||||
batch_size, padded_q_len, num_words = _topk_mask_shape(
|
||||
len(q_lens), max_seqlen_q, max_seqlen_k
|
||||
)
|
||||
words_needed = batch_size * padded_q_len * num_words
|
||||
if words_needed > topk_mask_workspace.numel():
|
||||
raise ValueError(
|
||||
f"Sparse MLA top-k mask needs {words_needed} int32 words (batch="
|
||||
f"{len(q_lens)}, q={max_seqlen_q}, k={max_seqlen_k}) but the "
|
||||
f"workspace holds {topk_mask_workspace.numel()}."
|
||||
)
|
||||
workspace_3d = topk_mask_workspace[:words_needed].view(
|
||||
batch_size, padded_q_len, num_words
|
||||
)
|
||||
dense_mask = _build_topk_mask(
|
||||
topk_per_req,
|
||||
q_lens,
|
||||
|
||||
Reference in New Issue
Block a user