[2/N][Attention] Enable masked MHA for sparse MLA prefills (#48770)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-07-31 06:01:24 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 864a87febb
commit 82ae4164ee
11 changed files with 855 additions and 26 deletions
+17 -1
View File
@@ -617,7 +617,7 @@ def main():
"--sparse-mla-mha-variants",
nargs="+",
default=None,
choices=["dense_mha", "mqa"],
choices=["dense_mha", "masked_mha", "mqa"],
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
)
@@ -748,6 +748,9 @@ def main():
args.sparse_mla_mha_variants = yaml_config.get(
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
)
args.sparse_mla_masked_mha_max_seq_len = yaml_config.get(
"sparse_mla_masked_mha_max_seq_len", None
)
# Parameter sweep configuration
if "parameter_sweep" in yaml_config:
@@ -1119,6 +1122,7 @@ def main():
console.print(f"Prefill backend: {prefill_backend}")
available_variants = [
("dense_mha", False, "dense"),
("masked_mha", False, "masked"),
("mqa", True, "auto"),
]
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
@@ -1139,6 +1143,9 @@ def main():
]
else:
variants = available_variants
masked_mha_max_seq_len = getattr(
args, "sparse_mla_masked_mha_max_seq_len", None
)
formatter = ResultsFormatter(console)
total = 0
for spec in args.batch_specs:
@@ -1148,6 +1155,10 @@ def main():
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
) or (
variant_label == "masked_mha"
and masked_mha_max_seq_len is not None
and q_len > masked_mha_max_seq_len
):
continue
total += len(backends)
@@ -1161,6 +1172,10 @@ def main():
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
) or (
variant_label == "masked_mha"
and masked_mha_max_seq_len is not None
and q_len > masked_mha_max_seq_len
):
continue
config = BenchmarkConfig(
@@ -1190,6 +1205,7 @@ def main():
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
prefill_backend=prefill_backend,
sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len,
)
# run_mla_benchmark needs the real backend name
+2 -1
View File
@@ -299,8 +299,9 @@ class BenchmarkConfig:
num_kv_splits: int | None = None # CUTLASS MLA
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
sparse_mla_mha_mode: str = "auto" # "auto", "dense", or "masked"
sparse_mla_dense_mha_max_seq_len: int | None = None
sparse_mla_masked_mha_max_seq_len: int | None = None
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
@@ -0,0 +1,123 @@
# Sparse MLA benchmark: forward_mha vs forward_mqa crossover plot
#
# Dense pure-prefill sweep to generate a smooth speedup curve. The 100 points
# run from q32 through q262144, approximately even on the log2 scale, and include
# routing boundaries plus extra samples around the 6k-8k crossover region.
# Dense MHA is only benchmarked through q32768 because it dominates runtime above
# that range.
#
# Usage:
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
mode: mha_vs_mqa
model:
name: "deepseek-v3"
num_layers: 60
num_q_heads: 16
num_kv_heads: 1
head_dim: 576
kv_lora_rank: 512
qk_nope_head_dim: 128
qk_rope_head_dim: 64
v_head_dim: 128
block_size: 128
max_model_len: 262144
batch_specs:
- "q32"
- "q36"
- "q40"
- "q45"
- "q50"
- "q56"
- "q62"
- "q70"
- "q78"
- "q87"
- "q97"
- "q109"
- "q122"
- "q136"
- "q152"
- "q170"
- "q190"
- "q212"
- "q237"
- "q255"
- "q256"
- "q257"
- "q265"
- "q296"
- "q331"
- "q370"
- "q413"
- "q462"
- "q516"
- "q577"
- "q645"
- "q721"
- "q806"
- "q901"
- "q1007"
- "q1125"
- "q1257"
- "q1405"
- "q1571"
- "q1756"
- "q1962"
- "q2048"
- "q2049"
- "q2193"
- "q2451"
- "q2740"
- "q3062"
- "q3422"
- "q3825"
- "q4275"
- "q4778"
- "q5340"
- "q5854"
- "q5969"
- "q6000"
- "q6144"
- "q6367"
- "q6671"
- "q6925"
- "q7168"
- "q7456"
- "q7532"
- "q8192"
- "q8193"
- "q8333"
- "q9314"
- "q10410"
- "q11635"
- "q13004"
- "q14534"
- "q16244"
- "q18156"
- "q20292"
- "q22680"
- "q25349"
- "q28332"
- "q31665"
- "q32768"
- "q32769"
- "q35391"
- "q39556"
- "q44210"
- "q49413"
- "q55227"
- "q61726"
- "q65536"
backends:
- FLASHMLA_SPARSE
device: "cuda:0"
profile_memory: true
sparse_mla_dense_mha_max_seq_len: 25349
sparse_mla_masked_mha_max_seq_len: 8192
sparse_mla_mha_variants: ["dense_mha", "masked_mha", "mqa"]
sparse_mla_topk_pattern: "random"
@@ -931,10 +931,17 @@ def _run_single_benchmark(
prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
fused_output = output_scale is not None and fuse_quant_op
mha_mode = getattr(config, "sparse_mla_mha_mode", "auto")
# Build forward function (runs a single decode/prefill pass)
def forward_fn():
results = []
impl._sparse_mla_force_masked_mha = ( # type: ignore[attr-defined]
mha_mode == "masked"
)
impl._sparse_mla_force_dense_mha = ( # type: ignore[attr-defined]
mha_mode == "dense"
)
if has_decode:
results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
if has_prefill:
@@ -39,7 +39,7 @@ else()
FetchContent_Declare(
vllm-flash-attn
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
GIT_TAG ed4b7342bc8f0489dd9b649d5288867e35fc6a32
GIT_TAG ec4a882d70b19235c824848db944d84a6cb8f9ee
GIT_PROGRESS TRUE
# Don't share the vllm-flash-attn build between build types
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
+26 -6
View File
@@ -862,6 +862,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
PREFILL_BATCH_SPECS = {
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
"masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
"masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
}
@@ -878,7 +880,7 @@ def test_sparse_backend_prefill_correctness(
kv_cache_dtype,
workspace_init,
):
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
backend_cls = FlashMLASparseBackend
batch_spec = PREFILL_BATCH_SPECS[batch_name]
@@ -892,7 +894,8 @@ def test_sparse_backend_prefill_correctness(
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 512
masked_mha = batch_name.startswith("masked_mha")
topk_tokens = 200 if masked_mha else 512
max_seqlen = max(batch_spec.seq_lens)
total_cache_tokens = sum(batch_spec.seq_lens)
@@ -918,6 +921,7 @@ def test_sparse_backend_prefill_correctness(
model_type="deepseek_v2",
)
model_config.dtype = dtype
model_config.model_arch_config.total_num_attention_heads = num_heads
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads, model_config
)
@@ -942,13 +946,14 @@ def test_sparse_backend_prefill_correctness(
# Compute dense reference outputs.
total_query_tokens = sum(query_lens)
sparse_indices = torch.zeros(
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
sparse_indices = torch.full(
(total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device
)
all_q, all_kv_c_new, all_k_pe_new = [], [], []
kv_c_contexts, k_pe_contexts = [], []
reference_outputs = []
global_token_idx = 0
for i in range(batch_spec.batch_size):
s_len = seq_lens[i]
@@ -981,8 +986,15 @@ def test_sparse_backend_prefill_correctness(
for j in range(q_len):
attend_end = ctx_len + j + 1
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
k_attend = k_all[:attend_end] # (N, H, D_qk)
v_attend = v_all[:attend_end] # (N, H, D_v)
if masked_mha:
actual_topk = min(topk_tokens, attend_end)
attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
sparse_indices[global_token_idx, :actual_topk] = attend_indices
k_attend = k_all[attend_indices]
v_attend = v_all[attend_indices]
else:
k_attend = k_all[:attend_end] # (N, H, D_qk)
v_attend = v_all[:attend_end] # (N, H, D_v)
q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
@@ -993,6 +1005,7 @@ def test_sparse_backend_prefill_correctness(
)
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
global_token_idx += 1
all_q.append(q_mha)
all_kv_c_new.append(kv_c_full[ctx_len:])
@@ -1047,6 +1060,13 @@ def test_sparse_backend_prefill_correctness(
builder_cls = backend_cls.get_builder_cls()
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
if batch_name == "masked_mha_chunked_context":
builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
builder.chunked_prefill_workspace = torch.empty(
(builder.chunked_prefill_workspace_size, head_size),
dtype=dtype,
device=device,
)
# Drive the queries through the dense-MHA prefill path directly (the routing
# threshold would otherwise classify these short queries as MQA decodes).
builder.reorder_batch_threshold = 1
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
pytest.importorskip("cutlass")
from vllm.model_executor.layers.attention.sparse_mla_attention import (
_build_topk_mask,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_build_topk_mask_single_request_matches_generic_path() -> None:
topk = torch.tensor(
[[0, 31, 32, 63, -1], [1, 64, 127, -1, -1]],
dtype=torch.int32,
device="cuda",
)
num_words = (128 + 31) // 32
single_out = torch.zeros(1, 2, num_words, dtype=torch.int32, device="cuda")
generic_out = torch.zeros(2, 1, num_words, dtype=torch.int32, device="cuda")
single_req = _build_topk_mask([topk], [2], 2, 128, single_out)
generic = _build_topk_mask([topk[:1], topk[1:]], [1, 1], 1, 128, generic_out)
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
@@ -772,8 +772,25 @@ class MLAAttention(nn.Module, AttentionLayerBase):
num_mha_tokens = q.size(0) - num_mqa_tokens
if self.impl.is_sparse and num_mha_tokens > 0:
prefill_metadata = getattr(attn_metadata, "prefill", None)
if not getattr(prefill_metadata, "use_dense_mha", False):
prefill = getattr(attn_metadata, "prefill", None)
use_dense_mha = getattr(prefill, "use_dense_mha", False)
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
use_masked_mha = (
self.prefill_backend is not None
and self.impl.masked_mha_available # type: ignore[attr-defined]
and self.impl.dcp_world_size <= 1
and prefill is not None
and _use_masked_mha(
backend_name=self.attn_backend.get_name(),
tensor_parallel_size=self._vllm_config.parallel_config.tensor_parallel_size,
query_len=prefill.max_query_len,
seq_len=prefill_max_seq_len,
)
)
use_mha = (use_dense_mha or use_masked_mha) and not (
self._vllm_config.attention_config.sparse_mla_force_mqa
)
if not use_mha:
num_mqa_tokens = q.size(0)
num_mha_tokens = 0
@@ -781,6 +798,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
quant_key is not None
and self.prefill_backend is not None
and self.prefill_backend.supports_quant_output(quant_key)
and (
not self.impl.is_sparse
or attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
<= attn_metadata.topk_tokens # type: ignore[attr-defined]
)
and attn_metadata is not None
and attn_metadata.prefill is not None
and attn_metadata.prefill.chunked_context is None
@@ -1387,6 +1409,7 @@ class MLACommonPrefillMetadata:
seq_tot: list[int]
max_seq_lens: list[int]
seq_lens: torch.Tensor
context_lens: torch.Tensor
workspace: torch.Tensor
token_to_seq: torch.Tensor
chunk_total_token: list[int]
@@ -1408,10 +1431,9 @@ class MLACommonPrefillMetadata:
q_data_type: torch.dtype | None = None
output_dtype: torch.dtype | None = None
prefill_backend: MLAPrefillBackend | None = None
# Whether the prefill suffix is routed through dense MHA.
# Indexer scoring may be skipped only for a pure-prefill batch,
# since decode tokens still consume top-k indices.
query_lens_cpu: torch.Tensor | None = None
use_dense_mha: bool = False
topk_mask_workspace: torch.Tensor | None = None
@dataclass
@@ -1508,6 +1530,39 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
)
_DSV32_MASKED_MHA_THRESHOLDS: dict[str, dict[int, tuple[int | None, ...]]] = {
"FLASHMLA_SPARSE": {
1: (1536, 4096, None, None, None),
2: (512, 1024, 4096, None, None),
4: (512, 1024, 1536, 8192, None),
8: (512, 512, 1024, 2048, 16384),
},
"FLASHINFER_MLA_SPARSE": {
8: (512, 1024, 1024, 2048, 32768),
},
}
_DSV32_SEQ_LEN_BUCKETS = (2048, 4096, 8192, 16384, 32768)
def _use_masked_mha(
*,
backend_name: str,
tensor_parallel_size: int,
query_len: int,
seq_len: int,
) -> bool:
thresholds = _DSV32_MASKED_MHA_THRESHOLDS.get(backend_name, {}).get(
tensor_parallel_size
)
if thresholds is None:
return False
for bucket_idx, bucket_seq_len in enumerate(_DSV32_SEQ_LEN_BUCKETS):
if seq_len <= bucket_seq_len:
min_query_len = thresholds[bucket_idx]
return min_query_len is not None and query_len >= min_query_len
return False
@functools.cache
def backend_supports_prefill_query_quantization() -> bool:
"""Check if the selected MLA prefill backend supports query quantization.
@@ -1682,6 +1737,7 @@ def build_mla_chunked_context_metadata(
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
seq_lens=chunk_seq_lens,
context_lens=context_lens_cpu.to(device, non_blocking=True),
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
chunk_total_token=chunk_total_token.tolist(),
workspace=chunked_prefill_workspace,
@@ -1705,6 +1761,7 @@ def build_mla_chunked_context_metadata(
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
seq_lens=chunk_seq_lens,
context_lens=context_lens_cpu.to(device, non_blocking=True),
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
chunk_total_token=chunk_total_token,
workspace=chunked_prefill_workspace,
@@ -1,29 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
from shutil import which
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
import numpy as np
import torch
from vllm.distributed import get_dcp_group
from vllm import _custom_ops as ops
from vllm.distributed import (
get_dcp_group,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBaseImpl,
MLACommonMetadata,
MLACommonPrefillMetadata,
build_mla_chunked_context_metadata,
get_mla_dims,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.torch_utils import np_to_pinned_tensor
from vllm.v1.attention.backend import (
AttentionMetadata,
AttentionMetadataBuilder,
)
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -35,6 +40,32 @@ logger = init_logger(__name__)
T = TypeVar("T", bound=AttentionMetadata)
GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
def _is_masked_mha_available(
num_heads_total: int,
kv_lora_rank: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
kv_cache_dtype: str,
) -> bool:
"""Check if masked MHA can ever fire for this model configuration."""
if not current_platform.is_device_capability_family(100):
return False
if (
num_heads_total != 128
or kv_lora_rank != 512
or qk_nope_head_dim != 128
or qk_rope_head_dim != 64
or v_head_dim != 128
):
return False
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
fa_version = get_flash_attn_version(head_size=qk_head_dim, head_size_v=v_head_dim)
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
metadata_cls: type[T]
@@ -86,6 +117,20 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dtype=self.model_config.dtype,
device=device,
)
self.topk_mask_workspace: torch.Tensor | None = None
if _is_masked_mha_available(
self.model_config.model_arch_config.total_num_attention_heads,
self.mla_dims.kv_lora_rank,
self.mla_dims.qk_nope_head_dim,
self.mla_dims.qk_rope_head_dim,
self.mla_dims.v_head_dim,
vllm_config.cache_config.cache_dtype,
):
self.topk_mask_workspace = torch.zeros(
GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
dtype=torch.int32,
device=device,
)
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend
@@ -194,6 +239,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
query_start_loc=prefill_query_start_loc,
max_query_len=prefill_max_query_len,
query_lens_cpu=prefill_query_lens_cpu,
chunked_context=self._build_chunked_context_fields(
common_attn_metadata,
num_decodes,
@@ -207,6 +253,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
prefill_max_seq_len <= self.topk_tokens
and not self.vllm_config.attention_config.sparse_mla_force_mqa
),
topk_mask_workspace=self.topk_mask_workspace,
)
self._prefill_backend.prepare_metadata(prefill)
@@ -260,9 +307,139 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
)
@triton.jit
def _scatter_topk_kernel(
mask_ptr,
topk_ptr,
cu_q_lens_ptr,
num_words: tl.constexpr,
mask_row_stride: tl.constexpr,
num_topk: tl.constexpr,
topk_stride: tl.constexpr,
max_q_len: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_WORDS: tl.constexpr,
):
row_idx = tl.program_id(0)
row_ptr = mask_ptr + row_idx * mask_row_stride
word_offsets = tl.arange(0, BLOCK_WORDS)
tl.store(
row_ptr + word_offsets,
tl.zeros([BLOCK_WORDS], dtype=tl.int32),
mask=word_offsets < num_words,
)
req_idx = row_idx // max_q_len
q_local = row_idx % max_q_len
q_start = tl.load(cu_q_lens_ptr + req_idx)
q_len = tl.load(cu_q_lens_ptr + req_idx + 1) - q_start
if q_local < q_len:
src_row = q_start + q_local
offsets = tl.arange(0, BLOCK_TOPK)
in_range = offsets < num_topk
indices = tl.load(
topk_ptr + src_row * topk_stride + offsets,
mask=in_range,
other=-1,
)
valid = in_range & (indices >= 0)
word_indices = indices >> 5
bits = (1 << (indices & 31)).to(tl.int32)
tl.atomic_or(row_ptr + word_indices, bits, mask=valid)
@triton.jit
def _scatter_topk_single_req_kernel(
mask_ptr,
topk_ptr,
num_words: tl.constexpr,
mask_row_stride: tl.constexpr,
num_topk: tl.constexpr,
topk_stride: tl.constexpr,
total_q: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_WORDS: tl.constexpr,
):
row_idx = tl.program_id(0)
row_ptr = mask_ptr + row_idx * mask_row_stride
word_offsets = tl.arange(0, BLOCK_WORDS)
tl.store(
row_ptr + word_offsets,
tl.zeros([BLOCK_WORDS], dtype=tl.int32),
mask=word_offsets < num_words,
)
if row_idx < total_q:
offsets = tl.arange(0, BLOCK_TOPK)
in_range = offsets < num_topk
indices = tl.load(
topk_ptr + row_idx * topk_stride + offsets,
mask=in_range,
other=-1,
)
valid = in_range & (indices >= 0)
word_indices = indices >> 5
bits = (1 << (indices & 31)).to(tl.int32)
tl.atomic_or(row_ptr + word_indices, bits, mask=valid)
def _build_topk_mask(
topk_indices_per_req: list[torch.Tensor],
q_lens: list[int],
max_q_len: int,
max_seq_len: int,
out: torch.Tensor,
) -> torch.Tensor:
"""Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``."""
batch_size = len(q_lens)
num_words = (max_seq_len + 31) // 32
total_rows = batch_size * max_q_len
if total_rows == 0:
return out[:batch_size, :max_q_len, :num_words]
total_q = sum(q_lens)
mask_row_stride = out.stride(-2)
block_words = triton.next_power_of_2(num_words)
if batch_size == 1:
topk_packed = topk_indices_per_req[0]
num_topk = topk_packed.shape[1]
_scatter_topk_single_req_kernel[(max_q_len,)](
out,
topk_packed,
num_words=num_words,
mask_row_stride=mask_row_stride,
num_topk=num_topk,
topk_stride=topk_packed.stride(0),
total_q=total_q,
BLOCK_TOPK=triton.next_power_of_2(num_topk),
BLOCK_WORDS=block_words,
)
return out[:1, :max_q_len, :num_words]
topk_packed = torch.cat(topk_indices_per_req, dim=0)
num_topk = topk_packed.shape[1]
q_lens_tensor = np_to_pinned_tensor(np.asarray(q_lens, dtype=np.int32)).to(
out.device, non_blocking=True
)
cu_q_lens = out.new_zeros(batch_size + 1)
torch.cumsum(q_lens_tensor, dim=0, out=cu_q_lens[1:])
_scatter_topk_kernel[(total_rows,)](
out,
topk_packed,
cu_q_lens,
num_words=num_words,
mask_row_stride=mask_row_stride,
num_topk=num_topk,
topk_stride=topk_packed.stride(0),
max_q_len=max_q_len,
BLOCK_TOPK=triton.next_power_of_2(num_topk),
BLOCK_WORDS=block_words,
)
return out[:batch_size, :max_q_len, :num_words]
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
"""Sparse MLA base: shared dense-MHA prefill (from MLACommonBaseImpl) plus the
sparse top-k MQA decode path. Subclasses implement forward_mqa."""
"""Sparse MLA base with dense and masked-MHA prefill paths."""
is_sparse = True
@@ -278,7 +455,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
# MLA-specific
q_lora_rank: int | None,
kv_lora_rank: int,
qk_nope_head_dim: int,
@@ -312,7 +488,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
if indexer is not None
else topk_indices_buffer
)
self._use_flashinfer_concat_mla_k = (
has_flashinfer()
and which("ninja") is not None
@@ -320,3 +495,335 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
and (self.qk_nope_head_dim == 128)
and (self.qk_rope_head_dim == 64)
)
self.masked_mha_available = _is_masked_mha_available(
num_heads_total=num_heads * get_tensor_model_parallel_world_size(),
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_cache_dtype=kv_cache_dtype,
)
@staticmethod
def _slice_topk_per_req(
topk_all: torch.Tensor,
q_lens: list[int],
) -> list[torch.Tensor]:
topk_per_req = []
offset = 0
for q_len in q_lens:
topk_per_req.append(topk_all[offset : offset + q_len])
offset += q_len
return topk_per_req
@staticmethod
def _remap_topk_to_ranges(
topk_per_req: list[torch.Tensor],
range_starts: list[int] | torch.Tensor,
range_lens: list[int],
) -> list[torch.Tensor]:
remapped = []
for topk, start, length in zip(topk_per_req, range_starts, range_lens):
valid = (topk >= start) & (topk < start + length)
remapped.append(torch.where(valid, topk - start, -1))
return remapped
def _project_kv(
self, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1,
self.num_heads,
self.qk_nope_head_dim + self.v_head_dim,
)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
return self._concat_k_nope_k_pe(k_nope, k_pe), v
@staticmethod
def _try_build_global_mask(
topk_per_req: list[torch.Tensor],
q_lens: list[int],
max_query_len: int,
max_seq_len: int,
topk_mask_workspace: torch.Tensor,
) -> torch.Tensor | None:
"""Build a full-sequence top-k mask if it fits within the budget.
When the mask fits, it is reused across the suffix and all context
chunks, avoiding per-chunk mask rebuilds. Returns None when the
mask is too large, signalling the caller to fall back to per-chunk
index remapping.
"""
batch_size = len(q_lens)
tile_m = 128 if max_query_len <= 128 else 256
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
num_words_padded = (max_seq_len + 31) // 32 + 1
needed = batch_size * padded_q_len * num_words_padded
if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES:
return None
mask = topk_mask_workspace[:needed].view(
batch_size, padded_q_len, num_words_padded
)
_build_topk_mask(
topk_per_req,
q_lens,
padded_q_len,
max_seq_len,
mask,
)
return mask
def _run_masked_mha(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
cu_seqlens_q: torch.Tensor,
cu_seqlens_k: torch.Tensor,
max_seqlen_q: int,
max_seqlen_k: int,
topk_per_req: list[torch.Tensor],
q_lens: list[int],
causal: bool,
return_softmax_lse: bool = False,
dense_mask: torch.Tensor | None = None,
key_starts: torch.Tensor | None = None,
topk_mask_workspace: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
from vllm.model_executor.layers.attention.sparse_mla_mask import (
dense_mask_mod,
offset_dense_mask_mod,
)
from vllm.vllm_flash_attn import flash_attn_varlen_func
tile_m = 128 if max_seqlen_q <= 128 else 256
padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m
if dense_mask is None:
batch_size = len(q_lens)
num_words = (max_seqlen_k + 31) // 32
assert topk_mask_workspace is not None
workspace_3d = topk_mask_workspace[
: batch_size * padded_q_len * num_words
].view(batch_size, padded_q_len, num_words)
dense_mask = _build_topk_mask(
topk_per_req,
q_lens,
padded_q_len,
max_seqlen_k,
workspace_3d,
)
if key_starts is not None:
dense_mask[:, 0, -1].copy_(key_starts)
kwargs = {
"q": q,
"k": k,
"v": v,
"cu_seqlens_q": cu_seqlens_q,
"cu_seqlens_k": cu_seqlens_k,
"max_seqlen_q": max_seqlen_q,
"max_seqlen_k": max_seqlen_k,
"softmax_scale": self.scale,
"return_softmax_lse": return_softmax_lse,
"fa_version": 4,
"mask_mod": dense_mask_mod if key_starts is None else offset_dense_mask_mod,
"aux_tensors": [dense_mask],
"aux_tensor_leading_dims": [2],
"causal": causal,
}
return flash_attn_varlen_func(**kwargs)
def _compute_context_mha(
self,
q: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
prefill_metadata: MLACommonPrefillMetadata,
k_scale: torch.Tensor,
q_lens: list[int],
topk_per_req: list[torch.Tensor],
dense_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.dcp_world_size > 1:
raise NotImplementedError(
"Masked MHA with context does not yet support decode context "
"parallelism"
)
chunked_context = prefill_metadata.chunked_context
assert chunked_context is not None
use_global_mask = dense_mask is not None
output: torch.Tensor | None = None
output_lse: torch.Tensor | None = None
workspace = chunked_context.workspace
for i, toks in enumerate(chunked_context.seq_tot):
if toks == 0:
continue
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=chunked_context.cu_seq_lens[i],
token_to_seq=chunked_context.token_to_seq[i],
num_tokens=chunked_context.chunk_total_token[i],
kv_cache_dtype=self.kv_cache_dtype,
scale=k_scale,
seq_starts=chunked_context.starts[i],
)
chunk_kv_c = workspace[:toks, : self.kv_lora_rank]
chunk_k_pe = workspace[:toks, self.kv_lora_rank :].unsqueeze(1)
k, v = self._project_kv(chunk_kv_c, chunk_k_pe)
chunk_lens = chunked_context.seq_lens[i].tolist()
chunk_topk = (
topk_per_req
if use_global_mask
else self._remap_topk_to_ranges(
topk_per_req,
chunked_context.starts[i],
chunk_lens,
)
)
attn_out, lse = self._run_masked_mha(
q=q,
k=k,
v=v,
cu_seqlens_q=prefill_metadata.query_start_loc,
cu_seqlens_k=chunked_context.cu_seq_lens[i],
max_seqlen_q=prefill_metadata.max_query_len,
max_seqlen_k=chunked_context.max_seq_lens[i],
topk_per_req=chunk_topk,
q_lens=q_lens,
causal=False,
return_softmax_lse=True,
dense_mask=dense_mask,
key_starts=(chunked_context.starts[i] if use_global_mask else None),
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
)
if output is None:
output = attn_out
output_lse = lse
else:
assert output_lse is not None
merge_attn_states(
output=output,
output_lse=output_lse,
prefix_output=output,
prefix_lse=output_lse,
suffix_output=attn_out,
suffix_lse=lse,
)
assert output is not None and output_lse is not None
return output, output_lse
def forward_mha( # type: ignore[override]
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: T,
k_scale: torch.Tensor,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
) -> None:
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined]
force_dense = getattr(self, "_sparse_mla_force_dense_mha", False)
force_masked = getattr(self, "_sparse_mla_force_masked_mha", False)
if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked):
return super().forward_mha(
q,
kv_c_normed,
k_pe,
kv_c_and_k_pe_cache,
cast(MLACommonMetadata, attn_metadata),
k_scale,
output,
output_scale,
)
assert output_scale is None
assert self.masked_mha_available
prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined]
assert prefill_metadata is not None
assert prefill_metadata.query_lens_cpu is not None
assert self.topk_indices_buffer is not None
q_lens = prefill_metadata.query_lens_cpu.tolist()
num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined]
topk_all = self.topk_indices_buffer[
num_decode_tokens : num_decode_tokens + q.shape[0]
]
topk_per_req = self._slice_topk_per_req(topk_all, q_lens)
k, v = self._project_kv(kv_c_normed, k_pe)
chunked_context = prefill_metadata.chunked_context
if chunked_context is None:
attn_out = self._run_masked_mha(
q=q,
k=k,
v=v,
cu_seqlens_q=prefill_metadata.query_start_loc,
cu_seqlens_k=prefill_metadata.query_start_loc,
max_seqlen_q=prefill_metadata.max_query_len,
max_seqlen_k=prefill_metadata.max_query_len,
topk_per_req=topk_per_req,
q_lens=q_lens,
causal=True,
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
)
assert isinstance(attn_out, torch.Tensor)
output.copy_(attn_out[..., : self.v_head_dim].flatten(start_dim=-2))
return
context_lens = chunked_context.seq_lens.sum(dim=0).tolist()
dense_mask = self._try_build_global_mask(
topk_per_req,
q_lens,
prefill_metadata.max_query_len,
prefill_max_seq_len,
prefill_metadata.topk_mask_workspace,
)
if dense_mask is not None:
suffix_topk = topk_per_req
else:
suffix_topk = self._remap_topk_to_ranges(topk_per_req, context_lens, q_lens)
suffix_output, suffix_lse = self._run_masked_mha(
q=q,
k=k,
v=v,
cu_seqlens_q=prefill_metadata.query_start_loc,
cu_seqlens_k=prefill_metadata.query_start_loc,
max_seqlen_q=prefill_metadata.max_query_len,
max_seqlen_k=prefill_metadata.max_query_len,
topk_per_req=suffix_topk,
q_lens=q_lens,
causal=True,
return_softmax_lse=True,
dense_mask=dense_mask,
key_starts=(
chunked_context.context_lens if dense_mask is not None else None
),
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
)
context_output, context_lse = self._compute_context_mha(
q=q,
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
prefill_metadata=prefill_metadata,
k_scale=k_scale,
q_lens=q_lens,
topk_per_req=topk_per_req,
dense_mask=dense_mask,
)
merge_attn_states(
output=output.view(-1, self.num_heads, self.v_head_dim),
prefix_output=context_output[..., : self.v_head_dim],
prefix_lse=context_lse,
suffix_output=suffix_output[..., : self.v_head_dim],
suffix_lse=suffix_lse,
prefill_tokens_with_context=chunked_context.prefill_tokens_with_context,
)
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import cutlass
import cutlass.cute as cute
from vllm.vllm_flash_attn.cute import utils
@cute.jit
def dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
kv_idx = utils.ssa_to_scalar(kv_idx)
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
dense_mask_mod.__vec_size__ = 32
@cute.jit
def offset_dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
offset_dense_mask_mod.__vec_size__ = 32
@@ -210,7 +210,9 @@ def flash_attn_varlen_func(
cp_tot_seqused_k=None,
# FA4 only
mask_mod=None,
block_sparse_tensors=None,
aux_tensors=None,
aux_tensor_leading_dims=None,
dynamic_causal: "torch.Tensor | None" = None,
):
"""dropout_p should be set to 0.0 during evaluation
@@ -385,6 +387,15 @@ def flash_attn_varlen_func(
)
elif fa_version == 4:
assert alibi_slopes is None, "Alibi is not supported in FA4"
if block_sparse_tensors is not None:
assert block_sparse_tensors.full_block_cnt is not None, (
"FA4 block_sparse_tensors must materialize empty full_block_cnt "
"instead of passing None"
)
assert block_sparse_tensors.full_block_idx is not None, (
"FA4 block_sparse_tensors must materialize empty full_block_idx "
"instead of passing None"
)
from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd
@@ -409,7 +420,9 @@ def flash_attn_varlen_func(
out=out,
learnable_sink=s_aux,
mask_mod=mask_mod,
block_sparse_tensors=block_sparse_tensors,
aux_tensors=aux_tensors,
aux_tensor_leading_dims=aux_tensor_leading_dims,
output_scale=output_scale,
)
else: