[2/N][Attention] Enable masked MHA for sparse MLA prefills (#48770)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
864a87febb
commit
82ae4164ee
@@ -617,7 +617,7 @@ def main():
|
||||
"--sparse-mla-mha-variants",
|
||||
nargs="+",
|
||||
default=None,
|
||||
choices=["dense_mha", "mqa"],
|
||||
choices=["dense_mha", "masked_mha", "mqa"],
|
||||
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
|
||||
)
|
||||
|
||||
@@ -748,6 +748,9 @@ def main():
|
||||
args.sparse_mla_mha_variants = yaml_config.get(
|
||||
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
|
||||
)
|
||||
args.sparse_mla_masked_mha_max_seq_len = yaml_config.get(
|
||||
"sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
|
||||
# Parameter sweep configuration
|
||||
if "parameter_sweep" in yaml_config:
|
||||
@@ -1119,6 +1122,7 @@ def main():
|
||||
console.print(f"Prefill backend: {prefill_backend}")
|
||||
available_variants = [
|
||||
("dense_mha", False, "dense"),
|
||||
("masked_mha", False, "masked"),
|
||||
("mqa", True, "auto"),
|
||||
]
|
||||
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
|
||||
@@ -1139,6 +1143,9 @@ def main():
|
||||
]
|
||||
else:
|
||||
variants = available_variants
|
||||
masked_mha_max_seq_len = getattr(
|
||||
args, "sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
formatter = ResultsFormatter(console)
|
||||
total = 0
|
||||
for spec in args.batch_specs:
|
||||
@@ -1148,6 +1155,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
total += len(backends)
|
||||
@@ -1161,6 +1172,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
config = BenchmarkConfig(
|
||||
@@ -1190,6 +1205,7 @@ def main():
|
||||
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
|
||||
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
|
||||
prefill_backend=prefill_backend,
|
||||
sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len,
|
||||
)
|
||||
|
||||
# run_mla_benchmark needs the real backend name
|
||||
|
||||
@@ -299,8 +299,9 @@ class BenchmarkConfig:
|
||||
num_kv_splits: int | None = None # CUTLASS MLA
|
||||
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
||||
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
|
||||
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
|
||||
sparse_mla_mha_mode: str = "auto" # "auto", "dense", or "masked"
|
||||
sparse_mla_dense_mha_max_seq_len: int | None = None
|
||||
sparse_mla_masked_mha_max_seq_len: int | None = None
|
||||
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
|
||||
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# Sparse MLA benchmark: forward_mha vs forward_mqa crossover plot
|
||||
#
|
||||
# Dense pure-prefill sweep to generate a smooth speedup curve. The 100 points
|
||||
# run from q32 through q262144, approximately even on the log2 scale, and include
|
||||
# routing boundaries plus extra samples around the 6k-8k crossover region.
|
||||
# Dense MHA is only benchmarked through q32768 because it dominates runtime above
|
||||
# that range.
|
||||
#
|
||||
# Usage:
|
||||
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
|
||||
|
||||
mode: mha_vs_mqa
|
||||
|
||||
model:
|
||||
name: "deepseek-v3"
|
||||
num_layers: 60
|
||||
num_q_heads: 16
|
||||
num_kv_heads: 1
|
||||
head_dim: 576
|
||||
kv_lora_rank: 512
|
||||
qk_nope_head_dim: 128
|
||||
qk_rope_head_dim: 64
|
||||
v_head_dim: 128
|
||||
block_size: 128
|
||||
max_model_len: 262144
|
||||
|
||||
batch_specs:
|
||||
- "q32"
|
||||
- "q36"
|
||||
- "q40"
|
||||
- "q45"
|
||||
- "q50"
|
||||
- "q56"
|
||||
- "q62"
|
||||
- "q70"
|
||||
- "q78"
|
||||
- "q87"
|
||||
- "q97"
|
||||
- "q109"
|
||||
- "q122"
|
||||
- "q136"
|
||||
- "q152"
|
||||
- "q170"
|
||||
- "q190"
|
||||
- "q212"
|
||||
- "q237"
|
||||
- "q255"
|
||||
- "q256"
|
||||
- "q257"
|
||||
- "q265"
|
||||
- "q296"
|
||||
- "q331"
|
||||
- "q370"
|
||||
- "q413"
|
||||
- "q462"
|
||||
- "q516"
|
||||
- "q577"
|
||||
- "q645"
|
||||
- "q721"
|
||||
- "q806"
|
||||
- "q901"
|
||||
- "q1007"
|
||||
- "q1125"
|
||||
- "q1257"
|
||||
- "q1405"
|
||||
- "q1571"
|
||||
- "q1756"
|
||||
- "q1962"
|
||||
- "q2048"
|
||||
- "q2049"
|
||||
- "q2193"
|
||||
- "q2451"
|
||||
- "q2740"
|
||||
- "q3062"
|
||||
- "q3422"
|
||||
- "q3825"
|
||||
- "q4275"
|
||||
- "q4778"
|
||||
- "q5340"
|
||||
- "q5854"
|
||||
- "q5969"
|
||||
- "q6000"
|
||||
- "q6144"
|
||||
- "q6367"
|
||||
- "q6671"
|
||||
- "q6925"
|
||||
- "q7168"
|
||||
- "q7456"
|
||||
- "q7532"
|
||||
- "q8192"
|
||||
- "q8193"
|
||||
- "q8333"
|
||||
- "q9314"
|
||||
- "q10410"
|
||||
- "q11635"
|
||||
- "q13004"
|
||||
- "q14534"
|
||||
- "q16244"
|
||||
- "q18156"
|
||||
- "q20292"
|
||||
- "q22680"
|
||||
- "q25349"
|
||||
- "q28332"
|
||||
- "q31665"
|
||||
- "q32768"
|
||||
- "q32769"
|
||||
- "q35391"
|
||||
- "q39556"
|
||||
- "q44210"
|
||||
- "q49413"
|
||||
- "q55227"
|
||||
- "q61726"
|
||||
- "q65536"
|
||||
|
||||
backends:
|
||||
- FLASHMLA_SPARSE
|
||||
|
||||
device: "cuda:0"
|
||||
profile_memory: true
|
||||
sparse_mla_dense_mha_max_seq_len: 25349
|
||||
sparse_mla_masked_mha_max_seq_len: 8192
|
||||
sparse_mla_mha_variants: ["dense_mha", "masked_mha", "mqa"]
|
||||
sparse_mla_topk_pattern: "random"
|
||||
@@ -931,10 +931,17 @@ def _run_single_benchmark(
|
||||
prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
|
||||
fused_output = output_scale is not None and fuse_quant_op
|
||||
mha_mode = getattr(config, "sparse_mla_mha_mode", "auto")
|
||||
|
||||
# Build forward function (runs a single decode/prefill pass)
|
||||
def forward_fn():
|
||||
results = []
|
||||
impl._sparse_mla_force_masked_mha = ( # type: ignore[attr-defined]
|
||||
mha_mode == "masked"
|
||||
)
|
||||
impl._sparse_mla_force_dense_mha = ( # type: ignore[attr-defined]
|
||||
mha_mode == "dense"
|
||||
)
|
||||
if has_decode:
|
||||
results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
|
||||
if has_prefill:
|
||||
|
||||
@@ -39,7 +39,7 @@ else()
|
||||
FetchContent_Declare(
|
||||
vllm-flash-attn
|
||||
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
|
||||
GIT_TAG ed4b7342bc8f0489dd9b649d5288867e35fc6a32
|
||||
GIT_TAG ec4a882d70b19235c824848db944d84a6cb8f9ee
|
||||
GIT_PROGRESS TRUE
|
||||
# Don't share the vllm-flash-attn build between build types
|
||||
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
|
||||
|
||||
@@ -862,6 +862,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
||||
PREFILL_BATCH_SPECS = {
|
||||
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
|
||||
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
|
||||
"masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
|
||||
"masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
|
||||
}
|
||||
|
||||
|
||||
@@ -878,7 +880,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
kv_cache_dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
|
||||
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
|
||||
backend_cls = FlashMLASparseBackend
|
||||
batch_spec = PREFILL_BATCH_SPECS[batch_name]
|
||||
|
||||
@@ -892,7 +894,8 @@ def test_sparse_backend_prefill_correctness(
|
||||
qk_rope_head_dim = 64
|
||||
v_head_dim = 128
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
topk_tokens = 512
|
||||
masked_mha = batch_name.startswith("masked_mha")
|
||||
topk_tokens = 200 if masked_mha else 512
|
||||
|
||||
max_seqlen = max(batch_spec.seq_lens)
|
||||
total_cache_tokens = sum(batch_spec.seq_lens)
|
||||
@@ -918,6 +921,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
model_config.dtype = dtype
|
||||
model_config.model_arch_config.total_num_attention_heads = num_heads
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads, model_config
|
||||
)
|
||||
@@ -942,13 +946,14 @@ def test_sparse_backend_prefill_correctness(
|
||||
|
||||
# Compute dense reference outputs.
|
||||
total_query_tokens = sum(query_lens)
|
||||
sparse_indices = torch.zeros(
|
||||
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
|
||||
sparse_indices = torch.full(
|
||||
(total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
all_q, all_kv_c_new, all_k_pe_new = [], [], []
|
||||
kv_c_contexts, k_pe_contexts = [], []
|
||||
reference_outputs = []
|
||||
global_token_idx = 0
|
||||
|
||||
for i in range(batch_spec.batch_size):
|
||||
s_len = seq_lens[i]
|
||||
@@ -981,8 +986,15 @@ def test_sparse_backend_prefill_correctness(
|
||||
for j in range(q_len):
|
||||
attend_end = ctx_len + j + 1
|
||||
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
|
||||
k_attend = k_all[:attend_end] # (N, H, D_qk)
|
||||
v_attend = v_all[:attend_end] # (N, H, D_v)
|
||||
if masked_mha:
|
||||
actual_topk = min(topk_tokens, attend_end)
|
||||
attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
|
||||
sparse_indices[global_token_idx, :actual_topk] = attend_indices
|
||||
k_attend = k_all[attend_indices]
|
||||
v_attend = v_all[attend_indices]
|
||||
else:
|
||||
k_attend = k_all[:attend_end] # (N, H, D_qk)
|
||||
v_attend = v_all[:attend_end] # (N, H, D_v)
|
||||
|
||||
q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
|
||||
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
|
||||
@@ -993,6 +1005,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
)
|
||||
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
|
||||
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
|
||||
global_token_idx += 1
|
||||
|
||||
all_q.append(q_mha)
|
||||
all_kv_c_new.append(kv_c_full[ctx_len:])
|
||||
@@ -1047,6 +1060,13 @@ def test_sparse_backend_prefill_correctness(
|
||||
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
if batch_name == "masked_mha_chunked_context":
|
||||
builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
|
||||
builder.chunked_prefill_workspace = torch.empty(
|
||||
(builder.chunked_prefill_workspace_size, head_size),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
# Drive the queries through the dense-MHA prefill path directly (the routing
|
||||
# threshold would otherwise classify these short queries as MQA decodes).
|
||||
builder.reorder_batch_threshold = 1
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("cutlass")
|
||||
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
_build_topk_mask,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_build_topk_mask_single_request_matches_generic_path() -> None:
|
||||
topk = torch.tensor(
|
||||
[[0, 31, 32, 63, -1], [1, 64, 127, -1, -1]],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
num_words = (128 + 31) // 32
|
||||
single_out = torch.zeros(1, 2, num_words, dtype=torch.int32, device="cuda")
|
||||
generic_out = torch.zeros(2, 1, num_words, dtype=torch.int32, device="cuda")
|
||||
single_req = _build_topk_mask([topk], [2], 2, 128, single_out)
|
||||
generic = _build_topk_mask([topk[:1], topk[1:]], [1, 1], 1, 128, generic_out)
|
||||
|
||||
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
|
||||
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
|
||||
@@ -772,8 +772,25 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill_metadata = getattr(attn_metadata, "prefill", None)
|
||||
if not getattr(prefill_metadata, "use_dense_mha", False):
|
||||
prefill = getattr(attn_metadata, "prefill", None)
|
||||
use_dense_mha = getattr(prefill, "use_dense_mha", False)
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
use_masked_mha = (
|
||||
self.prefill_backend is not None
|
||||
and self.impl.masked_mha_available # type: ignore[attr-defined]
|
||||
and self.impl.dcp_world_size <= 1
|
||||
and prefill is not None
|
||||
and _use_masked_mha(
|
||||
backend_name=self.attn_backend.get_name(),
|
||||
tensor_parallel_size=self._vllm_config.parallel_config.tensor_parallel_size,
|
||||
query_len=prefill.max_query_len,
|
||||
seq_len=prefill_max_seq_len,
|
||||
)
|
||||
)
|
||||
use_mha = (use_dense_mha or use_masked_mha) and not (
|
||||
self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
if not use_mha:
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
|
||||
@@ -781,6 +798,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
quant_key is not None
|
||||
and self.prefill_backend is not None
|
||||
and self.prefill_backend.supports_quant_output(quant_key)
|
||||
and (
|
||||
not self.impl.is_sparse
|
||||
or attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
<= attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
)
|
||||
and attn_metadata is not None
|
||||
and attn_metadata.prefill is not None
|
||||
and attn_metadata.prefill.chunked_context is None
|
||||
@@ -1387,6 +1409,7 @@ class MLACommonPrefillMetadata:
|
||||
seq_tot: list[int]
|
||||
max_seq_lens: list[int]
|
||||
seq_lens: torch.Tensor
|
||||
context_lens: torch.Tensor
|
||||
workspace: torch.Tensor
|
||||
token_to_seq: torch.Tensor
|
||||
chunk_total_token: list[int]
|
||||
@@ -1408,10 +1431,9 @@ class MLACommonPrefillMetadata:
|
||||
q_data_type: torch.dtype | None = None
|
||||
output_dtype: torch.dtype | None = None
|
||||
prefill_backend: MLAPrefillBackend | None = None
|
||||
# Whether the prefill suffix is routed through dense MHA.
|
||||
# Indexer scoring may be skipped only for a pure-prefill batch,
|
||||
# since decode tokens still consume top-k indices.
|
||||
query_lens_cpu: torch.Tensor | None = None
|
||||
use_dense_mha: bool = False
|
||||
topk_mask_workspace: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1508,6 +1530,39 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
)
|
||||
|
||||
|
||||
_DSV32_MASKED_MHA_THRESHOLDS: dict[str, dict[int, tuple[int | None, ...]]] = {
|
||||
"FLASHMLA_SPARSE": {
|
||||
1: (1536, 4096, None, None, None),
|
||||
2: (512, 1024, 4096, None, None),
|
||||
4: (512, 1024, 1536, 8192, None),
|
||||
8: (512, 512, 1024, 2048, 16384),
|
||||
},
|
||||
"FLASHINFER_MLA_SPARSE": {
|
||||
8: (512, 1024, 1024, 2048, 32768),
|
||||
},
|
||||
}
|
||||
_DSV32_SEQ_LEN_BUCKETS = (2048, 4096, 8192, 16384, 32768)
|
||||
|
||||
|
||||
def _use_masked_mha(
|
||||
*,
|
||||
backend_name: str,
|
||||
tensor_parallel_size: int,
|
||||
query_len: int,
|
||||
seq_len: int,
|
||||
) -> bool:
|
||||
thresholds = _DSV32_MASKED_MHA_THRESHOLDS.get(backend_name, {}).get(
|
||||
tensor_parallel_size
|
||||
)
|
||||
if thresholds is None:
|
||||
return False
|
||||
for bucket_idx, bucket_seq_len in enumerate(_DSV32_SEQ_LEN_BUCKETS):
|
||||
if seq_len <= bucket_seq_len:
|
||||
min_query_len = thresholds[bucket_idx]
|
||||
return min_query_len is not None and query_len >= min_query_len
|
||||
return False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def backend_supports_prefill_query_quantization() -> bool:
|
||||
"""Check if the selected MLA prefill backend supports query quantization.
|
||||
@@ -1682,6 +1737,7 @@ def build_mla_chunked_context_metadata(
|
||||
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token.tolist(),
|
||||
workspace=chunked_prefill_workspace,
|
||||
@@ -1705,6 +1761,7 @@ def build_mla_chunked_context_metadata(
|
||||
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token,
|
||||
workspace=chunked_prefill_workspace,
|
||||
|
||||
@@ -1,29 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
|
||||
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
build_mla_chunked_context_metadata,
|
||||
get_mla_dims,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.torch_utils import np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -35,6 +40,32 @@ logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
|
||||
|
||||
|
||||
def _is_masked_mha_available(
|
||||
num_heads_total: int,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_cache_dtype: str,
|
||||
) -> bool:
|
||||
"""Check if masked MHA can ever fire for this model configuration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False
|
||||
if (
|
||||
num_heads_total != 128
|
||||
or kv_lora_rank != 512
|
||||
or qk_nope_head_dim != 128
|
||||
or qk_rope_head_dim != 64
|
||||
or v_head_dim != 128
|
||||
):
|
||||
return False
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
fa_version = get_flash_attn_version(head_size=qk_head_dim, head_size_v=v_head_dim)
|
||||
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
|
||||
|
||||
|
||||
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
metadata_cls: type[T]
|
||||
@@ -86,6 +117,20 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dtype=self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.topk_mask_workspace: torch.Tensor | None = None
|
||||
if _is_masked_mha_available(
|
||||
self.model_config.model_arch_config.total_num_attention_heads,
|
||||
self.mla_dims.kv_lora_rank,
|
||||
self.mla_dims.qk_nope_head_dim,
|
||||
self.mla_dims.qk_rope_head_dim,
|
||||
self.mla_dims.v_head_dim,
|
||||
vllm_config.cache_config.cache_dtype,
|
||||
):
|
||||
self.topk_mask_workspace = torch.zeros(
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
@@ -194,6 +239,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
max_query_len=prefill_max_query_len,
|
||||
query_lens_cpu=prefill_query_lens_cpu,
|
||||
chunked_context=self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
@@ -207,6 +253,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
prefill_max_seq_len <= self.topk_tokens
|
||||
and not self.vllm_config.attention_config.sparse_mla_force_mqa
|
||||
),
|
||||
topk_mask_workspace=self.topk_mask_workspace,
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
@@ -260,9 +307,139 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_topk_kernel(
|
||||
mask_ptr,
|
||||
topk_ptr,
|
||||
cu_q_lens_ptr,
|
||||
num_words: tl.constexpr,
|
||||
mask_row_stride: tl.constexpr,
|
||||
num_topk: tl.constexpr,
|
||||
topk_stride: tl.constexpr,
|
||||
max_q_len: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
BLOCK_WORDS: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0)
|
||||
row_ptr = mask_ptr + row_idx * mask_row_stride
|
||||
word_offsets = tl.arange(0, BLOCK_WORDS)
|
||||
tl.store(
|
||||
row_ptr + word_offsets,
|
||||
tl.zeros([BLOCK_WORDS], dtype=tl.int32),
|
||||
mask=word_offsets < num_words,
|
||||
)
|
||||
|
||||
req_idx = row_idx // max_q_len
|
||||
q_local = row_idx % max_q_len
|
||||
q_start = tl.load(cu_q_lens_ptr + req_idx)
|
||||
q_len = tl.load(cu_q_lens_ptr + req_idx + 1) - q_start
|
||||
if q_local < q_len:
|
||||
src_row = q_start + q_local
|
||||
offsets = tl.arange(0, BLOCK_TOPK)
|
||||
in_range = offsets < num_topk
|
||||
indices = tl.load(
|
||||
topk_ptr + src_row * topk_stride + offsets,
|
||||
mask=in_range,
|
||||
other=-1,
|
||||
)
|
||||
valid = in_range & (indices >= 0)
|
||||
word_indices = indices >> 5
|
||||
bits = (1 << (indices & 31)).to(tl.int32)
|
||||
tl.atomic_or(row_ptr + word_indices, bits, mask=valid)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_topk_single_req_kernel(
|
||||
mask_ptr,
|
||||
topk_ptr,
|
||||
num_words: tl.constexpr,
|
||||
mask_row_stride: tl.constexpr,
|
||||
num_topk: tl.constexpr,
|
||||
topk_stride: tl.constexpr,
|
||||
total_q: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
BLOCK_WORDS: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0)
|
||||
row_ptr = mask_ptr + row_idx * mask_row_stride
|
||||
word_offsets = tl.arange(0, BLOCK_WORDS)
|
||||
tl.store(
|
||||
row_ptr + word_offsets,
|
||||
tl.zeros([BLOCK_WORDS], dtype=tl.int32),
|
||||
mask=word_offsets < num_words,
|
||||
)
|
||||
if row_idx < total_q:
|
||||
offsets = tl.arange(0, BLOCK_TOPK)
|
||||
in_range = offsets < num_topk
|
||||
indices = tl.load(
|
||||
topk_ptr + row_idx * topk_stride + offsets,
|
||||
mask=in_range,
|
||||
other=-1,
|
||||
)
|
||||
valid = in_range & (indices >= 0)
|
||||
word_indices = indices >> 5
|
||||
bits = (1 << (indices & 31)).to(tl.int32)
|
||||
tl.atomic_or(row_ptr + word_indices, bits, mask=valid)
|
||||
|
||||
|
||||
def _build_topk_mask(
|
||||
topk_indices_per_req: list[torch.Tensor],
|
||||
q_lens: list[int],
|
||||
max_q_len: int,
|
||||
max_seq_len: int,
|
||||
out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``."""
|
||||
batch_size = len(q_lens)
|
||||
num_words = (max_seq_len + 31) // 32
|
||||
total_rows = batch_size * max_q_len
|
||||
if total_rows == 0:
|
||||
return out[:batch_size, :max_q_len, :num_words]
|
||||
|
||||
total_q = sum(q_lens)
|
||||
mask_row_stride = out.stride(-2)
|
||||
block_words = triton.next_power_of_2(num_words)
|
||||
|
||||
if batch_size == 1:
|
||||
topk_packed = topk_indices_per_req[0]
|
||||
num_topk = topk_packed.shape[1]
|
||||
_scatter_topk_single_req_kernel[(max_q_len,)](
|
||||
out,
|
||||
topk_packed,
|
||||
num_words=num_words,
|
||||
mask_row_stride=mask_row_stride,
|
||||
num_topk=num_topk,
|
||||
topk_stride=topk_packed.stride(0),
|
||||
total_q=total_q,
|
||||
BLOCK_TOPK=triton.next_power_of_2(num_topk),
|
||||
BLOCK_WORDS=block_words,
|
||||
)
|
||||
return out[:1, :max_q_len, :num_words]
|
||||
|
||||
topk_packed = torch.cat(topk_indices_per_req, dim=0)
|
||||
num_topk = topk_packed.shape[1]
|
||||
q_lens_tensor = np_to_pinned_tensor(np.asarray(q_lens, dtype=np.int32)).to(
|
||||
out.device, non_blocking=True
|
||||
)
|
||||
cu_q_lens = out.new_zeros(batch_size + 1)
|
||||
torch.cumsum(q_lens_tensor, dim=0, out=cu_q_lens[1:])
|
||||
_scatter_topk_kernel[(total_rows,)](
|
||||
out,
|
||||
topk_packed,
|
||||
cu_q_lens,
|
||||
num_words=num_words,
|
||||
mask_row_stride=mask_row_stride,
|
||||
num_topk=num_topk,
|
||||
topk_stride=topk_packed.stride(0),
|
||||
max_q_len=max_q_len,
|
||||
BLOCK_TOPK=triton.next_power_of_2(num_topk),
|
||||
BLOCK_WORDS=block_words,
|
||||
)
|
||||
return out[:batch_size, :max_q_len, :num_words]
|
||||
|
||||
|
||||
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
"""Sparse MLA base: shared dense-MHA prefill (from MLACommonBaseImpl) plus the
|
||||
sparse top-k MQA decode path. Subclasses implement forward_mqa."""
|
||||
"""Sparse MLA base with dense and masked-MHA prefill paths."""
|
||||
|
||||
is_sparse = True
|
||||
|
||||
@@ -278,7 +455,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA-specific
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
@@ -312,7 +488,6 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
if indexer is not None
|
||||
else topk_indices_buffer
|
||||
)
|
||||
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and which("ninja") is not None
|
||||
@@ -320,3 +495,335 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
and (self.qk_nope_head_dim == 128)
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
self.masked_mha_available = _is_masked_mha_available(
|
||||
num_heads_total=num_heads * get_tensor_model_parallel_world_size(),
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _slice_topk_per_req(
|
||||
topk_all: torch.Tensor,
|
||||
q_lens: list[int],
|
||||
) -> list[torch.Tensor]:
|
||||
topk_per_req = []
|
||||
offset = 0
|
||||
for q_len in q_lens:
|
||||
topk_per_req.append(topk_all[offset : offset + q_len])
|
||||
offset += q_len
|
||||
return topk_per_req
|
||||
|
||||
@staticmethod
|
||||
def _remap_topk_to_ranges(
|
||||
topk_per_req: list[torch.Tensor],
|
||||
range_starts: list[int] | torch.Tensor,
|
||||
range_lens: list[int],
|
||||
) -> list[torch.Tensor]:
|
||||
remapped = []
|
||||
for topk, start, length in zip(topk_per_req, range_starts, range_lens):
|
||||
valid = (topk >= start) & (topk < start + length)
|
||||
remapped.append(torch.where(valid, topk - start, -1))
|
||||
return remapped
|
||||
|
||||
def _project_kv(
|
||||
self, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1,
|
||||
self.num_heads,
|
||||
self.qk_nope_head_dim + self.v_head_dim,
|
||||
)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
return self._concat_k_nope_k_pe(k_nope, k_pe), v
|
||||
|
||||
@staticmethod
|
||||
def _try_build_global_mask(
|
||||
topk_per_req: list[torch.Tensor],
|
||||
q_lens: list[int],
|
||||
max_query_len: int,
|
||||
max_seq_len: int,
|
||||
topk_mask_workspace: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
"""Build a full-sequence top-k mask if it fits within the budget.
|
||||
|
||||
When the mask fits, it is reused across the suffix and all context
|
||||
chunks, avoiding per-chunk mask rebuilds. Returns None when the
|
||||
mask is too large, signalling the caller to fall back to per-chunk
|
||||
index remapping.
|
||||
"""
|
||||
batch_size = len(q_lens)
|
||||
tile_m = 128 if max_query_len <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
|
||||
num_words_padded = (max_seq_len + 31) // 32 + 1
|
||||
needed = batch_size * padded_q_len * num_words_padded
|
||||
if needed * torch.int32.itemsize > GLOBAL_TOPK_MASK_MAX_BYTES:
|
||||
return None
|
||||
|
||||
mask = topk_mask_workspace[:needed].view(
|
||||
batch_size, padded_q_len, num_words_padded
|
||||
)
|
||||
_build_topk_mask(
|
||||
topk_per_req,
|
||||
q_lens,
|
||||
padded_q_len,
|
||||
max_seq_len,
|
||||
mask,
|
||||
)
|
||||
return mask
|
||||
|
||||
def _run_masked_mha(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
cu_seqlens_k: torch.Tensor,
|
||||
max_seqlen_q: int,
|
||||
max_seqlen_k: int,
|
||||
topk_per_req: list[torch.Tensor],
|
||||
q_lens: list[int],
|
||||
causal: bool,
|
||||
return_softmax_lse: bool = False,
|
||||
dense_mask: torch.Tensor | None = None,
|
||||
key_starts: torch.Tensor | None = None,
|
||||
topk_mask_workspace: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
from vllm.model_executor.layers.attention.sparse_mla_mask import (
|
||||
dense_mask_mod,
|
||||
offset_dense_mask_mod,
|
||||
)
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
|
||||
tile_m = 128 if max_seqlen_q <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_seqlen_q, tile_m) * tile_m
|
||||
if dense_mask is None:
|
||||
batch_size = len(q_lens)
|
||||
num_words = (max_seqlen_k + 31) // 32
|
||||
assert topk_mask_workspace is not None
|
||||
workspace_3d = topk_mask_workspace[
|
||||
: batch_size * padded_q_len * num_words
|
||||
].view(batch_size, padded_q_len, num_words)
|
||||
dense_mask = _build_topk_mask(
|
||||
topk_per_req,
|
||||
q_lens,
|
||||
padded_q_len,
|
||||
max_seqlen_k,
|
||||
workspace_3d,
|
||||
)
|
||||
if key_starts is not None:
|
||||
dense_mask[:, 0, -1].copy_(key_starts)
|
||||
kwargs = {
|
||||
"q": q,
|
||||
"k": k,
|
||||
"v": v,
|
||||
"cu_seqlens_q": cu_seqlens_q,
|
||||
"cu_seqlens_k": cu_seqlens_k,
|
||||
"max_seqlen_q": max_seqlen_q,
|
||||
"max_seqlen_k": max_seqlen_k,
|
||||
"softmax_scale": self.scale,
|
||||
"return_softmax_lse": return_softmax_lse,
|
||||
"fa_version": 4,
|
||||
"mask_mod": dense_mask_mod if key_starts is None else offset_dense_mask_mod,
|
||||
"aux_tensors": [dense_mask],
|
||||
"aux_tensor_leading_dims": [2],
|
||||
"causal": causal,
|
||||
}
|
||||
|
||||
return flash_attn_varlen_func(**kwargs)
|
||||
|
||||
def _compute_context_mha(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
prefill_metadata: MLACommonPrefillMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
q_lens: list[int],
|
||||
topk_per_req: list[torch.Tensor],
|
||||
dense_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.dcp_world_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Masked MHA with context does not yet support decode context "
|
||||
"parallelism"
|
||||
)
|
||||
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
use_global_mask = dense_mask is not None
|
||||
output: torch.Tensor | None = None
|
||||
output_lse: torch.Tensor | None = None
|
||||
workspace = chunked_context.workspace
|
||||
|
||||
for i, toks in enumerate(chunked_context.seq_tot):
|
||||
if toks == 0:
|
||||
continue
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=chunked_context.cu_seq_lens[i],
|
||||
token_to_seq=chunked_context.token_to_seq[i],
|
||||
num_tokens=chunked_context.chunk_total_token[i],
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=chunked_context.starts[i],
|
||||
)
|
||||
|
||||
chunk_kv_c = workspace[:toks, : self.kv_lora_rank]
|
||||
chunk_k_pe = workspace[:toks, self.kv_lora_rank :].unsqueeze(1)
|
||||
k, v = self._project_kv(chunk_kv_c, chunk_k_pe)
|
||||
chunk_lens = chunked_context.seq_lens[i].tolist()
|
||||
chunk_topk = (
|
||||
topk_per_req
|
||||
if use_global_mask
|
||||
else self._remap_topk_to_ranges(
|
||||
topk_per_req,
|
||||
chunked_context.starts[i],
|
||||
chunk_lens,
|
||||
)
|
||||
)
|
||||
attn_out, lse = self._run_masked_mha(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=chunked_context.cu_seq_lens[i],
|
||||
max_seqlen_q=prefill_metadata.max_query_len,
|
||||
max_seqlen_k=chunked_context.max_seq_lens[i],
|
||||
topk_per_req=chunk_topk,
|
||||
q_lens=q_lens,
|
||||
causal=False,
|
||||
return_softmax_lse=True,
|
||||
dense_mask=dense_mask,
|
||||
key_starts=(chunked_context.starts[i] if use_global_mask else None),
|
||||
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = attn_out
|
||||
output_lse = lse
|
||||
else:
|
||||
assert output_lse is not None
|
||||
merge_attn_states(
|
||||
output=output,
|
||||
output_lse=output_lse,
|
||||
prefix_output=output,
|
||||
prefix_lse=output_lse,
|
||||
suffix_output=attn_out,
|
||||
suffix_lse=lse,
|
||||
)
|
||||
|
||||
assert output is not None and output_lse is not None
|
||||
return output, output_lse
|
||||
|
||||
def forward_mha( # type: ignore[override]
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: T,
|
||||
k_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
force_dense = getattr(self, "_sparse_mla_force_dense_mha", False)
|
||||
force_masked = getattr(self, "_sparse_mla_force_masked_mha", False)
|
||||
if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked):
|
||||
return super().forward_mha(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_c_and_k_pe_cache,
|
||||
cast(MLACommonMetadata, attn_metadata),
|
||||
k_scale,
|
||||
output,
|
||||
output_scale,
|
||||
)
|
||||
|
||||
assert output_scale is None
|
||||
assert self.masked_mha_available
|
||||
prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined]
|
||||
assert prefill_metadata is not None
|
||||
assert prefill_metadata.query_lens_cpu is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
|
||||
q_lens = prefill_metadata.query_lens_cpu.tolist()
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined]
|
||||
topk_all = self.topk_indices_buffer[
|
||||
num_decode_tokens : num_decode_tokens + q.shape[0]
|
||||
]
|
||||
topk_per_req = self._slice_topk_per_req(topk_all, q_lens)
|
||||
|
||||
k, v = self._project_kv(kv_c_normed, k_pe)
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
if chunked_context is None:
|
||||
attn_out = self._run_masked_mha(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=prefill_metadata.query_start_loc,
|
||||
max_seqlen_q=prefill_metadata.max_query_len,
|
||||
max_seqlen_k=prefill_metadata.max_query_len,
|
||||
topk_per_req=topk_per_req,
|
||||
q_lens=q_lens,
|
||||
causal=True,
|
||||
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
|
||||
)
|
||||
assert isinstance(attn_out, torch.Tensor)
|
||||
output.copy_(attn_out[..., : self.v_head_dim].flatten(start_dim=-2))
|
||||
return
|
||||
|
||||
context_lens = chunked_context.seq_lens.sum(dim=0).tolist()
|
||||
dense_mask = self._try_build_global_mask(
|
||||
topk_per_req,
|
||||
q_lens,
|
||||
prefill_metadata.max_query_len,
|
||||
prefill_max_seq_len,
|
||||
prefill_metadata.topk_mask_workspace,
|
||||
)
|
||||
if dense_mask is not None:
|
||||
suffix_topk = topk_per_req
|
||||
else:
|
||||
suffix_topk = self._remap_topk_to_ranges(topk_per_req, context_lens, q_lens)
|
||||
suffix_output, suffix_lse = self._run_masked_mha(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=prefill_metadata.query_start_loc,
|
||||
max_seqlen_q=prefill_metadata.max_query_len,
|
||||
max_seqlen_k=prefill_metadata.max_query_len,
|
||||
topk_per_req=suffix_topk,
|
||||
q_lens=q_lens,
|
||||
causal=True,
|
||||
return_softmax_lse=True,
|
||||
dense_mask=dense_mask,
|
||||
key_starts=(
|
||||
chunked_context.context_lens if dense_mask is not None else None
|
||||
),
|
||||
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
|
||||
)
|
||||
context_output, context_lse = self._compute_context_mha(
|
||||
q=q,
|
||||
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
|
||||
prefill_metadata=prefill_metadata,
|
||||
k_scale=k_scale,
|
||||
q_lens=q_lens,
|
||||
topk_per_req=topk_per_req,
|
||||
dense_mask=dense_mask,
|
||||
)
|
||||
merge_attn_states(
|
||||
output=output.view(-1, self.num_heads, self.v_head_dim),
|
||||
prefix_output=context_output[..., : self.v_head_dim],
|
||||
prefix_lse=context_lse,
|
||||
suffix_output=suffix_output[..., : self.v_head_dim],
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
|
||||
from vllm.vllm_flash_attn.cute import utils
|
||||
|
||||
|
||||
@cute.jit
|
||||
def dense_mask_mod(
|
||||
batch: cute.TensorSSA,
|
||||
head: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors: list,
|
||||
) -> cute.TensorSSA:
|
||||
dense_mask = aux_tensors[0]
|
||||
batch_idx = utils.ssa_to_scalar(batch)
|
||||
q_idx = utils.ssa_to_scalar(q_idx)
|
||||
kv_idx = utils.ssa_to_scalar(kv_idx)
|
||||
word_idx = kv_idx >> 5
|
||||
bit_idx = cutlass.Uint32(kv_idx & 31)
|
||||
word = dense_mask[batch_idx, q_idx, word_idx]
|
||||
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
|
||||
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
|
||||
return result.load()
|
||||
|
||||
|
||||
dense_mask_mod.__vec_size__ = 32
|
||||
|
||||
|
||||
@cute.jit
|
||||
def offset_dense_mask_mod(
|
||||
batch: cute.TensorSSA,
|
||||
head: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors: list,
|
||||
) -> cute.TensorSSA:
|
||||
dense_mask = aux_tensors[0]
|
||||
batch_idx = utils.ssa_to_scalar(batch)
|
||||
q_idx = utils.ssa_to_scalar(q_idx)
|
||||
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
|
||||
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
|
||||
word_idx = kv_idx >> 5
|
||||
bit_idx = cutlass.Uint32(kv_idx & 31)
|
||||
word = dense_mask[batch_idx, q_idx, word_idx]
|
||||
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
|
||||
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
|
||||
return result.load()
|
||||
|
||||
|
||||
offset_dense_mask_mod.__vec_size__ = 32
|
||||
@@ -210,7 +210,9 @@ def flash_attn_varlen_func(
|
||||
cp_tot_seqused_k=None,
|
||||
# FA4 only
|
||||
mask_mod=None,
|
||||
block_sparse_tensors=None,
|
||||
aux_tensors=None,
|
||||
aux_tensor_leading_dims=None,
|
||||
dynamic_causal: "torch.Tensor | None" = None,
|
||||
):
|
||||
"""dropout_p should be set to 0.0 during evaluation
|
||||
@@ -385,6 +387,15 @@ def flash_attn_varlen_func(
|
||||
)
|
||||
elif fa_version == 4:
|
||||
assert alibi_slopes is None, "Alibi is not supported in FA4"
|
||||
if block_sparse_tensors is not None:
|
||||
assert block_sparse_tensors.full_block_cnt is not None, (
|
||||
"FA4 block_sparse_tensors must materialize empty full_block_cnt "
|
||||
"instead of passing None"
|
||||
)
|
||||
assert block_sparse_tensors.full_block_idx is not None, (
|
||||
"FA4 block_sparse_tensors must materialize empty full_block_idx "
|
||||
"instead of passing None"
|
||||
)
|
||||
|
||||
from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd
|
||||
|
||||
@@ -409,7 +420,9 @@ def flash_attn_varlen_func(
|
||||
out=out,
|
||||
learnable_sink=s_aux,
|
||||
mask_mod=mask_mod,
|
||||
block_sparse_tensors=block_sparse_tensors,
|
||||
aux_tensors=aux_tensors,
|
||||
aux_tensor_leading_dims=aux_tensor_leading_dims,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user