diff --git a/benchmarks/attention_benchmarks/benchmark.py b/benchmarks/attention_benchmarks/benchmark.py index a4941b4a18..8f01be94d2 100644 --- a/benchmarks/attention_benchmarks/benchmark.py +++ b/benchmarks/attention_benchmarks/benchmark.py @@ -617,7 +617,7 @@ def main(): "--sparse-mla-mha-variants", nargs="+", default=None, - choices=["dense_mha", "mqa"], + choices=["dense_mha", "masked_mha", "mqa"], help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.", ) @@ -748,6 +748,9 @@ def main(): args.sparse_mla_mha_variants = yaml_config.get( "sparse_mla_mha_variants", args.sparse_mla_mha_variants ) + args.sparse_mla_masked_mha_max_seq_len = yaml_config.get( + "sparse_mla_masked_mha_max_seq_len", None + ) # Parameter sweep configuration if "parameter_sweep" in yaml_config: @@ -1119,6 +1122,7 @@ def main(): console.print(f"Prefill backend: {prefill_backend}") available_variants = [ ("dense_mha", False, "dense"), + ("masked_mha", False, "masked"), ("mqa", True, "auto"), ] requested_variants = getattr(args, "sparse_mla_mha_variants", None) @@ -1139,6 +1143,9 @@ def main(): ] else: variants = available_variants + masked_mha_max_seq_len = getattr( + args, "sparse_mla_masked_mha_max_seq_len", None + ) formatter = ResultsFormatter(console) total = 0 for spec in args.batch_specs: @@ -1148,6 +1155,10 @@ def main(): variant_label == "dense_mha" and dense_mha_max_seq_len is not None and q_len > dense_mha_max_seq_len + ) or ( + variant_label == "masked_mha" + and masked_mha_max_seq_len is not None + and q_len > masked_mha_max_seq_len ): continue total += len(backends) @@ -1161,6 +1172,10 @@ def main(): variant_label == "dense_mha" and dense_mha_max_seq_len is not None and q_len > dense_mha_max_seq_len + ) or ( + variant_label == "masked_mha" + and masked_mha_max_seq_len is not None + and q_len > masked_mha_max_seq_len ): continue config = BenchmarkConfig( @@ -1190,6 +1205,7 @@ def main(): sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len, sparse_mla_topk_pattern=sparse_mla_topk_pattern, prefill_backend=prefill_backend, + sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len, ) # run_mla_benchmark needs the real backend name diff --git a/benchmarks/attention_benchmarks/common.py b/benchmarks/attention_benchmarks/common.py index b8ca5739c5..45858f9148 100644 --- a/benchmarks/attention_benchmarks/common.py +++ b/benchmarks/attention_benchmarks/common.py @@ -299,8 +299,9 @@ class BenchmarkConfig: num_kv_splits: int | None = None # CUTLASS MLA reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA - sparse_mla_mha_mode: str = "auto" # "auto" or "dense" + sparse_mla_mha_mode: str = "auto" # "auto", "dense", or "masked" sparse_mla_dense_mha_max_seq_len: int | None = None + sparse_mla_masked_mha_max_seq_len: int | None = None sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window" num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled) diff --git a/benchmarks/attention_benchmarks/configs/mla_sparse_masked_mha_vs_mqa.yaml b/benchmarks/attention_benchmarks/configs/mla_sparse_masked_mha_vs_mqa.yaml new file mode 100644 index 0000000000..a02b7f5ba0 --- /dev/null +++ b/benchmarks/attention_benchmarks/configs/mla_sparse_masked_mha_vs_mqa.yaml @@ -0,0 +1,123 @@ +# Sparse MLA benchmark: forward_mha vs forward_mqa crossover plot +# +# Dense pure-prefill sweep to generate a smooth speedup curve. The 100 points +# run from q32 through q262144, approximately even on the log2 scale, and include +# routing boundaries plus extra samples around the 6k-8k crossover region. +# Dense MHA is only benchmarked through q32768 because it dominates runtime above +# that range. +# +# Usage: +# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml + +mode: mha_vs_mqa + +model: + name: "deepseek-v3" + num_layers: 60 + num_q_heads: 16 + num_kv_heads: 1 + head_dim: 576 + kv_lora_rank: 512 + qk_nope_head_dim: 128 + qk_rope_head_dim: 64 + v_head_dim: 128 + block_size: 128 + max_model_len: 262144 + +batch_specs: + - "q32" + - "q36" + - "q40" + - "q45" + - "q50" + - "q56" + - "q62" + - "q70" + - "q78" + - "q87" + - "q97" + - "q109" + - "q122" + - "q136" + - "q152" + - "q170" + - "q190" + - "q212" + - "q237" + - "q255" + - "q256" + - "q257" + - "q265" + - "q296" + - "q331" + - "q370" + - "q413" + - "q462" + - "q516" + - "q577" + - "q645" + - "q721" + - "q806" + - "q901" + - "q1007" + - "q1125" + - "q1257" + - "q1405" + - "q1571" + - "q1756" + - "q1962" + - "q2048" + - "q2049" + - "q2193" + - "q2451" + - "q2740" + - "q3062" + - "q3422" + - "q3825" + - "q4275" + - "q4778" + - "q5340" + - "q5854" + - "q5969" + - "q6000" + - "q6144" + - "q6367" + - "q6671" + - "q6925" + - "q7168" + - "q7456" + - "q7532" + - "q8192" + - "q8193" + - "q8333" + - "q9314" + - "q10410" + - "q11635" + - "q13004" + - "q14534" + - "q16244" + - "q18156" + - "q20292" + - "q22680" + - "q25349" + - "q28332" + - "q31665" + - "q32768" + - "q32769" + - "q35391" + - "q39556" + - "q44210" + - "q49413" + - "q55227" + - "q61726" + - "q65536" + +backends: + - FLASHMLA_SPARSE + +device: "cuda:0" +profile_memory: true +sparse_mla_dense_mha_max_seq_len: 25349 +sparse_mla_masked_mha_max_seq_len: 8192 +sparse_mla_mha_variants: ["dense_mha", "masked_mha", "mqa"] +sparse_mla_topk_pattern: "random" diff --git a/benchmarks/attention_benchmarks/mla_runner.py b/benchmarks/attention_benchmarks/mla_runner.py index f78aca3bce..6d56fb11ac 100644 --- a/benchmarks/attention_benchmarks/mla_runner.py +++ b/benchmarks/attention_benchmarks/mla_runner.py @@ -931,10 +931,17 @@ def _run_single_benchmark( prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR) fused_output = output_scale is not None and fuse_quant_op + mha_mode = getattr(config, "sparse_mla_mha_mode", "auto") # Build forward function (runs a single decode/prefill pass) def forward_fn(): results = [] + impl._sparse_mla_force_masked_mha = ( # type: ignore[attr-defined] + mha_mode == "masked" + ) + impl._sparse_mla_force_dense_mha = ( # type: ignore[attr-defined] + mha_mode == "dense" + ) if has_decode: results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer)) if has_prefill: diff --git a/cmake/external_projects/vllm_flash_attn.cmake b/cmake/external_projects/vllm_flash_attn.cmake index 6b559d3725..15a4f10b7a 100644 --- a/cmake/external_projects/vllm_flash_attn.cmake +++ b/cmake/external_projects/vllm_flash_attn.cmake @@ -39,7 +39,7 @@ else() FetchContent_Declare( vllm-flash-attn GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git - GIT_TAG ed4b7342bc8f0489dd9b649d5288867e35fc6a32 + GIT_TAG ec4a882d70b19235c824848db944d84a6cb8f9ee GIT_PROGRESS TRUE # Don't share the vllm-flash-attn build between build types BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 3fe5df918a..0c7b645c14 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -862,6 +862,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected): PREFILL_BATCH_SPECS = { "short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]), "short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]), + "masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]), + "masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]), } @@ -878,7 +880,7 @@ def test_sparse_backend_prefill_correctness( kv_cache_dtype, workspace_init, ): - """Test single-pass FA4 dense forward_mha for sparse MLA prefill.""" + """Test single-pass dense and masked MHA for sparse MLA prefill.""" backend_cls = FlashMLASparseBackend batch_spec = PREFILL_BATCH_SPECS[batch_name] @@ -892,7 +894,8 @@ def test_sparse_backend_prefill_correctness( qk_rope_head_dim = 64 v_head_dim = 128 head_size = kv_lora_rank + qk_rope_head_dim - topk_tokens = 512 + masked_mha = batch_name.startswith("masked_mha") + topk_tokens = 200 if masked_mha else 512 max_seqlen = max(batch_spec.seq_lens) total_cache_tokens = sum(batch_spec.seq_lens) @@ -918,6 +921,7 @@ def test_sparse_backend_prefill_correctness( model_type="deepseek_v2", ) model_config.dtype = dtype + model_config.model_arch_config.total_num_attention_heads = num_heads model_config.get_num_attention_heads = MethodType( lambda self, parallel_config: num_heads, model_config ) @@ -942,13 +946,14 @@ def test_sparse_backend_prefill_correctness( # Compute dense reference outputs. total_query_tokens = sum(query_lens) - sparse_indices = torch.zeros( - total_query_tokens, topk_tokens, dtype=torch.int32, device=device + sparse_indices = torch.full( + (total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device ) all_q, all_kv_c_new, all_k_pe_new = [], [], [] kv_c_contexts, k_pe_contexts = [], [] reference_outputs = [] + global_token_idx = 0 for i in range(batch_spec.batch_size): s_len = seq_lens[i] @@ -981,8 +986,15 @@ def test_sparse_backend_prefill_correctness( for j in range(q_len): attend_end = ctx_len + j + 1 q_tok = q_mha[j : j + 1] # (1, H, D_qk) - k_attend = k_all[:attend_end] # (N, H, D_qk) - v_attend = v_all[:attend_end] # (N, H, D_v) + if masked_mha: + actual_topk = min(topk_tokens, attend_end) + attend_indices = torch.randperm(attend_end, device=device)[:actual_topk] + sparse_indices[global_token_idx, :actual_topk] = attend_indices + k_attend = k_all[attend_indices] + v_attend = v_all[attend_indices] + else: + k_attend = k_all[:attend_end] # (N, H, D_qk) + v_attend = v_all[:attend_end] # (N, H, D_v) q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float() k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float() @@ -993,6 +1005,7 @@ def test_sparse_backend_prefill_correctness( ) out = out.transpose(1, 2).squeeze(0) # (1, H, D_v) reference_outputs.append(out.to(dtype).flatten(start_dim=-2)) + global_token_idx += 1 all_q.append(q_mha) all_kv_c_new.append(kv_c_full[ctx_len:]) @@ -1047,6 +1060,13 @@ def test_sparse_backend_prefill_correctness( builder_cls = backend_cls.get_builder_cls() builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device) + if batch_name == "masked_mha_chunked_context": + builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size + builder.chunked_prefill_workspace = torch.empty( + (builder.chunked_prefill_workspace_size, head_size), + dtype=dtype, + device=device, + ) # Drive the queries through the dense-MHA prefill path directly (the routing # threshold would otherwise classify these short queries as MQA decodes). builder.reorder_batch_threshold = 1 diff --git a/tests/v1/attention/test_sparse_mla_mask.py b/tests/v1/attention/test_sparse_mla_mask.py new file mode 100644 index 0000000000..3ecc179be4 --- /dev/null +++ b/tests/v1/attention/test_sparse_mla_mask.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +pytest.importorskip("cutlass") + +from vllm.model_executor.layers.attention.sparse_mla_attention import ( + _build_topk_mask, +) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_build_topk_mask_single_request_matches_generic_path() -> None: + topk = torch.tensor( + [[0, 31, 32, 63, -1], [1, 64, 127, -1, -1]], + dtype=torch.int32, + device="cuda", + ) + + num_words = (128 + 31) // 32 + single_out = torch.zeros(1, 2, num_words, dtype=torch.int32, device="cuda") + generic_out = torch.zeros(2, 1, num_words, dtype=torch.int32, device="cuda") + single_req = _build_topk_mask([topk], [2], 2, 128, single_out) + generic = _build_topk_mask([topk[:1], topk[1:]], [1, 1], 1, 128, generic_out) + + torch.testing.assert_close(single_req[0, 0], generic[0, 0]) + torch.testing.assert_close(single_req[0, 1], generic[1, 0]) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 5292dd9c9a..3f737bed58 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -772,8 +772,25 @@ class MLAAttention(nn.Module, AttentionLayerBase): num_mha_tokens = q.size(0) - num_mqa_tokens if self.impl.is_sparse and num_mha_tokens > 0: - prefill_metadata = getattr(attn_metadata, "prefill", None) - if not getattr(prefill_metadata, "use_dense_mha", False): + prefill = getattr(attn_metadata, "prefill", None) + use_dense_mha = getattr(prefill, "use_dense_mha", False) + prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined] + use_masked_mha = ( + self.prefill_backend is not None + and self.impl.masked_mha_available # type: ignore[attr-defined] + and self.impl.dcp_world_size <= 1 + and prefill is not None + and _use_masked_mha( + backend_name=self.attn_backend.get_name(), + tensor_parallel_size=self._vllm_config.parallel_config.tensor_parallel_size, + query_len=prefill.max_query_len, + seq_len=prefill_max_seq_len, + ) + ) + use_mha = (use_dense_mha or use_masked_mha) and not ( + self._vllm_config.attention_config.sparse_mla_force_mqa + ) + if not use_mha: num_mqa_tokens = q.size(0) num_mha_tokens = 0 @@ -781,6 +798,11 @@ class MLAAttention(nn.Module, AttentionLayerBase): quant_key is not None and self.prefill_backend is not None and self.prefill_backend.supports_quant_output(quant_key) + and ( + not self.impl.is_sparse + or attn_metadata.prefill_max_seq_len # type: ignore[attr-defined] + <= attn_metadata.topk_tokens # type: ignore[attr-defined] + ) and attn_metadata is not None and attn_metadata.prefill is not None and attn_metadata.prefill.chunked_context is None @@ -1387,6 +1409,7 @@ class MLACommonPrefillMetadata: seq_tot: list[int] max_seq_lens: list[int] seq_lens: torch.Tensor + context_lens: torch.Tensor workspace: torch.Tensor token_to_seq: torch.Tensor chunk_total_token: list[int] @@ -1408,10 +1431,9 @@ class MLACommonPrefillMetadata: q_data_type: torch.dtype | None = None output_dtype: torch.dtype | None = None prefill_backend: MLAPrefillBackend | None = None - # Whether the prefill suffix is routed through dense MHA. - # Indexer scoring may be skipped only for a pure-prefill batch, - # since decode tokens still consume top-k indices. + query_lens_cpu: torch.Tensor | None = None use_dense_mha: bool = False + topk_mask_workspace: torch.Tensor | None = None @dataclass @@ -1508,6 +1530,39 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims: ) +_DSV32_MASKED_MHA_THRESHOLDS: dict[str, dict[int, tuple[int | None, ...]]] = { + "FLASHMLA_SPARSE": { + 1: (1536, 4096, None, None, None), + 2: (512, 1024, 4096, None, None), + 4: (512, 1024, 1536, 8192, None), + 8: (512, 512, 1024, 2048, 16384), + }, + "FLASHINFER_MLA_SPARSE": { + 8: (512, 1024, 1024, 2048, 32768), + }, +} +_DSV32_SEQ_LEN_BUCKETS = (2048, 4096, 8192, 16384, 32768) + + +def _use_masked_mha( + *, + backend_name: str, + tensor_parallel_size: int, + query_len: int, + seq_len: int, +) -> bool: + thresholds = _DSV32_MASKED_MHA_THRESHOLDS.get(backend_name, {}).get( + tensor_parallel_size + ) + if thresholds is None: + return False + for bucket_idx, bucket_seq_len in enumerate(_DSV32_SEQ_LEN_BUCKETS): + if seq_len <= bucket_seq_len: + min_query_len = thresholds[bucket_idx] + return min_query_len is not None and query_len >= min_query_len + return False + + @functools.cache def backend_supports_prefill_query_quantization() -> bool: """Check if the selected MLA prefill backend supports query quantization. @@ -1682,6 +1737,7 @@ def build_mla_chunked_context_metadata( seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(), max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(), seq_lens=chunk_seq_lens, + context_lens=context_lens_cpu.to(device, non_blocking=True), token_to_seq=token_to_seq_cpu.to(device, non_blocking=True), chunk_total_token=chunk_total_token.tolist(), workspace=chunked_prefill_workspace, @@ -1705,6 +1761,7 @@ def build_mla_chunked_context_metadata( seq_tot=chunk_seq_lens.sum(dim=1).tolist(), max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(), seq_lens=chunk_seq_lens, + context_lens=context_lens_cpu.to(device, non_blocking=True), token_to_seq=token_to_seq_cpu.to(device, non_blocking=True), chunk_total_token=chunk_total_token, workspace=chunked_prefill_workspace, diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index 1463f9fb35..90934d2892 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -1,29 +1,34 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Shared forward_mha implementation and metadata builder for sparse MLA backends.""" +"""Shared MHA implementation and metadata builder for sparse MLA backends.""" from shutil import which -from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast import numpy as np import torch -from vllm.distributed import get_dcp_group +from vllm import _custom_ops as ops +from vllm.distributed import ( + get_dcp_group, + get_tensor_model_parallel_world_size, +) from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( MLACommonBaseImpl, + MLACommonMetadata, MLACommonPrefillMetadata, build_mla_chunked_context_metadata, get_mla_dims, ) from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton from vllm.utils.flashinfer import has_flashinfer -from vllm.utils.torch_utils import np_to_pinned_tensor -from vllm.v1.attention.backend import ( - AttentionMetadata, - AttentionMetadataBuilder, -) +from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor +from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder +from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.utils import split_decodes_and_prefills +from vllm.v1.attention.ops.merge_attn_states import merge_attn_states if TYPE_CHECKING: from vllm.config import VllmConfig @@ -35,6 +40,32 @@ logger = init_logger(__name__) T = TypeVar("T", bound=AttentionMetadata) +GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB + + +def _is_masked_mha_available( + num_heads_total: int, + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + kv_cache_dtype: str, +) -> bool: + """Check if masked MHA can ever fire for this model configuration.""" + if not current_platform.is_device_capability_family(100): + return False + if ( + num_heads_total != 128 + or kv_lora_rank != 512 + or qk_nope_head_dim != 128 + or qk_rope_head_dim != 64 + or v_head_dim != 128 + ): + return False + qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + fa_version = get_flash_attn_version(head_size=qk_head_dim, head_size_v=v_head_dim) + return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype) + class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): metadata_cls: type[T] @@ -86,6 +117,20 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): dtype=self.model_config.dtype, device=device, ) + self.topk_mask_workspace: torch.Tensor | None = None + if _is_masked_mha_available( + self.model_config.model_arch_config.total_num_attention_heads, + self.mla_dims.kv_lora_rank, + self.mla_dims.qk_nope_head_dim, + self.mla_dims.qk_rope_head_dim, + self.mla_dims.v_head_dim, + vllm_config.cache_config.cache_dtype, + ): + self.topk_mask_workspace = torch.zeros( + GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize, + dtype=torch.int32, + device=device, + ) layer_prefill_backend = vllm_config.compilation_config.static_forward_context[ layer_names[0] ].prefill_backend @@ -194,6 +239,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...], query_start_loc=prefill_query_start_loc, max_query_len=prefill_max_query_len, + query_lens_cpu=prefill_query_lens_cpu, chunked_context=self._build_chunked_context_fields( common_attn_metadata, num_decodes, @@ -207,6 +253,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): prefill_max_seq_len <= self.topk_tokens and not self.vllm_config.attention_config.sparse_mla_force_mqa ), + topk_mask_workspace=self.topk_mask_workspace, ) self._prefill_backend.prepare_metadata(prefill) @@ -260,9 +307,139 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): ) +@triton.jit +def _scatter_topk_kernel( + mask_ptr, + topk_ptr, + cu_q_lens_ptr, + num_words: tl.constexpr, + mask_row_stride: tl.constexpr, + num_topk: tl.constexpr, + topk_stride: tl.constexpr, + max_q_len: tl.constexpr, + BLOCK_TOPK: tl.constexpr, + BLOCK_WORDS: tl.constexpr, +): + row_idx = tl.program_id(0) + row_ptr = mask_ptr + row_idx * mask_row_stride + word_offsets = tl.arange(0, BLOCK_WORDS) + tl.store( + row_ptr + word_offsets, + tl.zeros([BLOCK_WORDS], dtype=tl.int32), + mask=word_offsets < num_words, + ) + + req_idx = row_idx // max_q_len + q_local = row_idx % max_q_len + q_start = tl.load(cu_q_lens_ptr + req_idx) + q_len = tl.load(cu_q_lens_ptr + req_idx + 1) - q_start + if q_local < q_len: + src_row = q_start + q_local + offsets = tl.arange(0, BLOCK_TOPK) + in_range = offsets < num_topk + indices = tl.load( + topk_ptr + src_row * topk_stride + offsets, + mask=in_range, + other=-1, + ) + valid = in_range & (indices >= 0) + word_indices = indices >> 5 + bits = (1 << (indices & 31)).to(tl.int32) + tl.atomic_or(row_ptr + word_indices, bits, mask=valid) + + +@triton.jit +def _scatter_topk_single_req_kernel( + mask_ptr, + topk_ptr, + num_words: tl.constexpr, + mask_row_stride: tl.constexpr, + num_topk: tl.constexpr, + topk_stride: tl.constexpr, + total_q: tl.constexpr, + BLOCK_TOPK: tl.constexpr, + BLOCK_WORDS: tl.constexpr, +): + row_idx = tl.program_id(0) + row_ptr = mask_ptr + row_idx * mask_row_stride + word_offsets = tl.arange(0, BLOCK_WORDS) + tl.store( + row_ptr + word_offsets, + tl.zeros([BLOCK_WORDS], dtype=tl.int32), + mask=word_offsets < num_words, + ) + if row_idx < total_q: + offsets = tl.arange(0, BLOCK_TOPK) + in_range = offsets < num_topk + indices = tl.load( + topk_ptr + row_idx * topk_stride + offsets, + mask=in_range, + other=-1, + ) + valid = in_range & (indices >= 0) + word_indices = indices >> 5 + bits = (1 << (indices & 31)).to(tl.int32) + tl.atomic_or(row_ptr + word_indices, bits, mask=valid) + + +def _build_topk_mask( + topk_indices_per_req: list[torch.Tensor], + q_lens: list[int], + max_q_len: int, + max_seq_len: int, + out: torch.Tensor, +) -> torch.Tensor: + """Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``.""" + batch_size = len(q_lens) + num_words = (max_seq_len + 31) // 32 + total_rows = batch_size * max_q_len + if total_rows == 0: + return out[:batch_size, :max_q_len, :num_words] + + total_q = sum(q_lens) + mask_row_stride = out.stride(-2) + block_words = triton.next_power_of_2(num_words) + + if batch_size == 1: + topk_packed = topk_indices_per_req[0] + num_topk = topk_packed.shape[1] + _scatter_topk_single_req_kernel[(max_q_len,)]( + out, + topk_packed, + num_words=num_words, + mask_row_stride=mask_row_stride, + num_topk=num_topk, + topk_stride=topk_packed.stride(0), + total_q=total_q, + BLOCK_TOPK=triton.next_power_of_2(num_topk), + BLOCK_WORDS=block_words, + ) + return out[:1, :max_q_len, :num_words] + + topk_packed = torch.cat(topk_indices_per_req, dim=0) + num_topk = topk_packed.shape[1] + q_lens_tensor = np_to_pinned_tensor(np.asarray(q_lens, dtype=np.int32)).to( + out.device, non_blocking=True + ) + cu_q_lens = out.new_zeros(batch_size + 1) + torch.cumsum(q_lens_tensor, dim=0, out=cu_q_lens[1:]) + _scatter_topk_kernel[(total_rows,)]( + out, + topk_packed, + cu_q_lens, + num_words=num_words, + mask_row_stride=mask_row_stride, + num_topk=num_topk, + topk_stride=topk_packed.stride(0), + max_q_len=max_q_len, + BLOCK_TOPK=triton.next_power_of_2(num_topk), + BLOCK_WORDS=block_words, + ) + return out[:batch_size, :max_q_len, :num_words] + + class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): - """Sparse MLA base: shared dense-MHA prefill (from MLACommonBaseImpl) plus the - sparse top-k MQA decode path. Subclasses implement forward_mqa.""" + """Sparse MLA base with dense and masked-MHA prefill paths.""" is_sparse = True @@ -278,7 +455,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): logits_soft_cap: float | None, attn_type: str, kv_sharing_target_layer_name: str | None, - # MLA-specific q_lora_rank: int | None, kv_lora_rank: int, qk_nope_head_dim: int, @@ -312,7 +488,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): if indexer is not None else topk_indices_buffer ) - self._use_flashinfer_concat_mla_k = ( has_flashinfer() and which("ninja") is not None @@ -320,3 +495,335 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): and (self.qk_nope_head_dim == 128) and (self.qk_rope_head_dim == 64) ) + self.masked_mha_available = _is_masked_mha_available( + num_heads_total=num_heads * get_tensor_model_parallel_world_size(), + kv_lora_rank=kv_lora_rank, + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=qk_rope_head_dim, + v_head_dim=v_head_dim, + kv_cache_dtype=kv_cache_dtype, + ) + + @staticmethod + def _slice_topk_per_req( + topk_all: torch.Tensor, + q_lens: list[int], + ) -> list[torch.Tensor]: + topk_per_req = [] + offset = 0 + for q_len in q_lens: + topk_per_req.append(topk_all[offset : offset + q_len]) + offset += q_len + return topk_per_req + + @staticmethod + def _remap_topk_to_ranges( + topk_per_req: list[torch.Tensor], + range_starts: list[int] | torch.Tensor, + range_lens: list[int], + ) -> list[torch.Tensor]: + remapped = [] + for topk, start, length in zip(topk_per_req, range_starts, range_lens): + valid = (topk >= start) & (topk < start + length) + remapped.append(torch.where(valid, topk - start, -1)) + return remapped + + def _project_kv( + self, kv_c_normed: torch.Tensor, k_pe: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + kv_nope = self.kv_b_proj(kv_c_normed)[0].view( + -1, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + return self._concat_k_nope_k_pe(k_nope, k_pe), v + + @staticmethod + def _try_build_global_mask( + topk_per_req: list[torch.Tensor], + q_lens: list[int], + max_query_len: int, + max_seq_len: int, + topk_mask_workspace: torch.Tensor, + ) -> torch.Tensor | None: + """Build a full-sequence top-k mask if it fits within the budget. + + When the mask fits, it is reused across the suffix and all context + chunks, avoiding per-chunk mask rebuilds. Returns None when the + mask is too large, signalling the caller to fall back to per-chunk + index remapping. + """ + batch_size = len(q_lens) + tile_m = 128 if max_query_len <= 128 else 256 + padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m + num_words_padded = (max_seq_len + 31) // 32 + 1 + needed = batch_size * padded_q_len * num_words_padded + if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES: + return None + + mask = topk_mask_workspace[:needed].view( + batch_size, padded_q_len, num_words_padded + ) + _build_topk_mask( + topk_per_req, + q_lens, + padded_q_len, + max_seq_len, + mask, + ) + return mask + + def _run_masked_mha( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + topk_per_req: list[torch.Tensor], + q_lens: list[int], + causal: bool, + return_softmax_lse: bool = False, + dense_mask: torch.Tensor | None = None, + key_starts: torch.Tensor | None = None, + topk_mask_workspace: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + from vllm.model_executor.layers.attention.sparse_mla_mask import ( + dense_mask_mod, + offset_dense_mask_mod, + ) + from vllm.vllm_flash_attn import flash_attn_varlen_func + + tile_m = 128 if max_seqlen_q <= 128 else 256 + padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m + if dense_mask is None: + batch_size = len(q_lens) + num_words = (max_seqlen_k + 31) // 32 + assert topk_mask_workspace is not None + workspace_3d = topk_mask_workspace[ + : batch_size * padded_q_len * num_words + ].view(batch_size, padded_q_len, num_words) + dense_mask = _build_topk_mask( + topk_per_req, + q_lens, + padded_q_len, + max_seqlen_k, + workspace_3d, + ) + if key_starts is not None: + dense_mask[:, 0, -1].copy_(key_starts) + kwargs = { + "q": q, + "k": k, + "v": v, + "cu_seqlens_q": cu_seqlens_q, + "cu_seqlens_k": cu_seqlens_k, + "max_seqlen_q": max_seqlen_q, + "max_seqlen_k": max_seqlen_k, + "softmax_scale": self.scale, + "return_softmax_lse": return_softmax_lse, + "fa_version": 4, + "mask_mod": dense_mask_mod if key_starts is None else offset_dense_mask_mod, + "aux_tensors": [dense_mask], + "aux_tensor_leading_dims": [2], + "causal": causal, + } + + return flash_attn_varlen_func(**kwargs) + + def _compute_context_mha( + self, + q: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + prefill_metadata: MLACommonPrefillMetadata, + k_scale: torch.Tensor, + q_lens: list[int], + topk_per_req: list[torch.Tensor], + dense_mask: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.dcp_world_size > 1: + raise NotImplementedError( + "Masked MHA with context does not yet support decode context " + "parallelism" + ) + + chunked_context = prefill_metadata.chunked_context + assert chunked_context is not None + use_global_mask = dense_mask is not None + output: torch.Tensor | None = None + output_lse: torch.Tensor | None = None + workspace = chunked_context.workspace + + for i, toks in enumerate(chunked_context.seq_tot): + if toks == 0: + continue + ops.gather_and_maybe_dequant_cache( + src_cache=kv_c_and_k_pe_cache, + dst=workspace, + block_table=prefill_metadata.block_table, + cu_seq_lens=chunked_context.cu_seq_lens[i], + token_to_seq=chunked_context.token_to_seq[i], + num_tokens=chunked_context.chunk_total_token[i], + kv_cache_dtype=self.kv_cache_dtype, + scale=k_scale, + seq_starts=chunked_context.starts[i], + ) + + chunk_kv_c = workspace[:toks, : self.kv_lora_rank] + chunk_k_pe = workspace[:toks, self.kv_lora_rank :].unsqueeze(1) + k, v = self._project_kv(chunk_kv_c, chunk_k_pe) + chunk_lens = chunked_context.seq_lens[i].tolist() + chunk_topk = ( + topk_per_req + if use_global_mask + else self._remap_topk_to_ranges( + topk_per_req, + chunked_context.starts[i], + chunk_lens, + ) + ) + attn_out, lse = self._run_masked_mha( + q=q, + k=k, + v=v, + cu_seqlens_q=prefill_metadata.query_start_loc, + cu_seqlens_k=chunked_context.cu_seq_lens[i], + max_seqlen_q=prefill_metadata.max_query_len, + max_seqlen_k=chunked_context.max_seq_lens[i], + topk_per_req=chunk_topk, + q_lens=q_lens, + causal=False, + return_softmax_lse=True, + dense_mask=dense_mask, + key_starts=(chunked_context.starts[i] if use_global_mask else None), + topk_mask_workspace=prefill_metadata.topk_mask_workspace, + ) + + if output is None: + output = attn_out + output_lse = lse + else: + assert output_lse is not None + merge_attn_states( + output=output, + output_lse=output_lse, + prefix_output=output, + prefix_lse=output_lse, + suffix_output=attn_out, + suffix_lse=lse, + ) + + assert output is not None and output_lse is not None + return output, output_lse + + def forward_mha( # type: ignore[override] + self, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: T, + k_scale: torch.Tensor, + output: torch.Tensor, + output_scale: torch.Tensor | None = None, + ) -> None: + prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined] + topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined] + force_dense = getattr(self, "_sparse_mla_force_dense_mha", False) + force_masked = getattr(self, "_sparse_mla_force_masked_mha", False) + if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked): + return super().forward_mha( + q, + kv_c_normed, + k_pe, + kv_c_and_k_pe_cache, + cast(MLACommonMetadata, attn_metadata), + k_scale, + output, + output_scale, + ) + + assert output_scale is None + assert self.masked_mha_available + prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined] + assert prefill_metadata is not None + assert prefill_metadata.query_lens_cpu is not None + assert self.topk_indices_buffer is not None + + q_lens = prefill_metadata.query_lens_cpu.tolist() + num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined] + topk_all = self.topk_indices_buffer[ + num_decode_tokens : num_decode_tokens + q.shape[0] + ] + topk_per_req = self._slice_topk_per_req(topk_all, q_lens) + + k, v = self._project_kv(kv_c_normed, k_pe) + chunked_context = prefill_metadata.chunked_context + if chunked_context is None: + attn_out = self._run_masked_mha( + q=q, + k=k, + v=v, + cu_seqlens_q=prefill_metadata.query_start_loc, + cu_seqlens_k=prefill_metadata.query_start_loc, + max_seqlen_q=prefill_metadata.max_query_len, + max_seqlen_k=prefill_metadata.max_query_len, + topk_per_req=topk_per_req, + q_lens=q_lens, + causal=True, + topk_mask_workspace=prefill_metadata.topk_mask_workspace, + ) + assert isinstance(attn_out, torch.Tensor) + output.copy_(attn_out[..., : self.v_head_dim].flatten(start_dim=-2)) + return + + context_lens = chunked_context.seq_lens.sum(dim=0).tolist() + dense_mask = self._try_build_global_mask( + topk_per_req, + q_lens, + prefill_metadata.max_query_len, + prefill_max_seq_len, + prefill_metadata.topk_mask_workspace, + ) + if dense_mask is not None: + suffix_topk = topk_per_req + else: + suffix_topk = self._remap_topk_to_ranges(topk_per_req, context_lens, q_lens) + suffix_output, suffix_lse = self._run_masked_mha( + q=q, + k=k, + v=v, + cu_seqlens_q=prefill_metadata.query_start_loc, + cu_seqlens_k=prefill_metadata.query_start_loc, + max_seqlen_q=prefill_metadata.max_query_len, + max_seqlen_k=prefill_metadata.max_query_len, + topk_per_req=suffix_topk, + q_lens=q_lens, + causal=True, + return_softmax_lse=True, + dense_mask=dense_mask, + key_starts=( + chunked_context.context_lens if dense_mask is not None else None + ), + topk_mask_workspace=prefill_metadata.topk_mask_workspace, + ) + context_output, context_lse = self._compute_context_mha( + q=q, + kv_c_and_k_pe_cache=kv_c_and_k_pe_cache, + prefill_metadata=prefill_metadata, + k_scale=k_scale, + q_lens=q_lens, + topk_per_req=topk_per_req, + dense_mask=dense_mask, + ) + merge_attn_states( + output=output.view(-1, self.num_heads, self.v_head_dim), + prefix_output=context_output[..., : self.v_head_dim], + prefix_lse=context_lse, + suffix_output=suffix_output[..., : self.v_head_dim], + suffix_lse=suffix_lse, + prefill_tokens_with_context=chunked_context.prefill_tokens_with_context, + ) diff --git a/vllm/model_executor/layers/attention/sparse_mla_mask.py b/vllm/model_executor/layers/attention/sparse_mla_mask.py new file mode 100644 index 0000000000..1cd151034c --- /dev/null +++ b/vllm/model_executor/layers/attention/sparse_mla_mask.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import cutlass +import cutlass.cute as cute + +from vllm.vllm_flash_attn.cute import utils + + +@cute.jit +def dense_mask_mod( + batch: cute.TensorSSA, + head: cute.TensorSSA, + q_idx: cute.TensorSSA, + kv_idx: cute.TensorSSA, + seqlen_info, + aux_tensors: list, +) -> cute.TensorSSA: + dense_mask = aux_tensors[0] + batch_idx = utils.ssa_to_scalar(batch) + q_idx = utils.ssa_to_scalar(q_idx) + kv_idx = utils.ssa_to_scalar(kv_idx) + word_idx = kv_idx >> 5 + bit_idx = cutlass.Uint32(kv_idx & 31) + word = dense_mask[batch_idx, q_idx, word_idx] + result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32) + result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx) + return result.load() + + +dense_mask_mod.__vec_size__ = 32 + + +@cute.jit +def offset_dense_mask_mod( + batch: cute.TensorSSA, + head: cute.TensorSSA, + q_idx: cute.TensorSSA, + kv_idx: cute.TensorSSA, + seqlen_info, + aux_tensors: list, +) -> cute.TensorSSA: + dense_mask = aux_tensors[0] + batch_idx = utils.ssa_to_scalar(batch) + q_idx = utils.ssa_to_scalar(q_idx) + key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1] + kv_idx = utils.ssa_to_scalar(kv_idx) + key_start + word_idx = kv_idx >> 5 + bit_idx = cutlass.Uint32(kv_idx & 31) + word = dense_mask[batch_idx, q_idx, word_idx] + result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32) + result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx) + return result.load() + + +offset_dense_mask_mod.__vec_size__ = 32 diff --git a/vllm/vllm_flash_attn/flash_attn_interface.py b/vllm/vllm_flash_attn/flash_attn_interface.py index fc4948d0e9..d3f8a36e70 100644 --- a/vllm/vllm_flash_attn/flash_attn_interface.py +++ b/vllm/vllm_flash_attn/flash_attn_interface.py @@ -210,7 +210,9 @@ def flash_attn_varlen_func( cp_tot_seqused_k=None, # FA4 only mask_mod=None, + block_sparse_tensors=None, aux_tensors=None, + aux_tensor_leading_dims=None, dynamic_causal: "torch.Tensor | None" = None, ): """dropout_p should be set to 0.0 during evaluation @@ -385,6 +387,15 @@ def flash_attn_varlen_func( ) elif fa_version == 4: assert alibi_slopes is None, "Alibi is not supported in FA4" + if block_sparse_tensors is not None: + assert block_sparse_tensors.full_block_cnt is not None, ( + "FA4 block_sparse_tensors must materialize empty full_block_cnt " + "instead of passing None" + ) + assert block_sparse_tensors.full_block_idx is not None, ( + "FA4 block_sparse_tensors must materialize empty full_block_idx " + "instead of passing None" + ) from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd @@ -409,7 +420,9 @@ def flash_attn_varlen_func( out=out, learnable_sink=s_aux, mask_mod=mask_mod, + block_sparse_tensors=block_sparse_tensors, aux_tensors=aux_tensors, + aux_tensor_leading_dims=aux_tensor_leading_dims, output_scale=output_scale, ) else: