[1/N] Add dense MHA path for sparse MLA short sequences (#47327)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-07-14 00:29:56 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 0b0ef8d7eb
commit c4f5cd60da
21 changed files with 1990 additions and 646 deletions
+68 -37
View File
@@ -4,8 +4,10 @@
"""Common utilities for attention benchmarking."""
import csv
import gc
import json
import math
from collections.abc import Sequence
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
@@ -44,10 +46,13 @@ def run_do_bench(
kwargs: dict[str, Any] = {"return_mode": "all"}
if use_cuda_graphs:
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
gc.collect()
torch.accelerator.empty_cache()
else:
if warmup_ms is not None:
kwargs["warmup"] = warmup_ms
result = triton.testing.do_bench(benchmark_fn, **kwargs)
torch.accelerator.synchronize()
return result
@@ -91,42 +96,6 @@ except ImportError:
AttentionLayerBase = object # Fallback
class MockKVBProj:
"""Mock KV projection layer for MLA prefill mode.
Mimics ColumnParallelLinear behavior for kv_b_proj in MLA backends.
Projects kv_c_normed to [qk_nope_head_dim + v_head_dim] per head.
"""
def __init__(self, num_heads: int, qk_nope_head_dim: int, v_head_dim: int):
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.v_head_dim = v_head_dim
self.out_dim = qk_nope_head_dim + v_head_dim
self.weight = torch.empty(0, dtype=torch.bfloat16)
def __call__(self, x: torch.Tensor) -> tuple[torch.Tensor]:
"""
Project kv_c_normed to output space.
Args:
x: Input tensor [num_tokens, kv_lora_rank]
Returns:
Tuple containing output tensor
[num_tokens, num_heads, qk_nope_head_dim + v_head_dim]
"""
num_tokens = x.shape[0]
result = torch.randn(
num_tokens,
self.num_heads,
self.out_dim,
device=x.device,
dtype=x.dtype,
)
return (result,) # Return as tuple to match ColumnParallelLinear API
class MockIndexer:
"""Mock Indexer for sparse MLA backends.
@@ -158,6 +127,60 @@ class MockIndexer:
)
self.topk_indices_buffer[:num_tokens] = indices
def fill_indices(
self,
num_tokens: int,
max_kv_len: int,
pattern: str = "random",
requests: Sequence[Any] | None = None,
):
if pattern == "random":
self.fill_random_indices(num_tokens, max_kv_len)
return
if pattern == "prefix":
indices = torch.arange(
self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
indices = (indices % max_kv_len).expand(num_tokens, -1)
self.topk_indices_buffer[:num_tokens] = indices
return
if pattern == "sliding_window":
if requests is None:
start = max(max_kv_len - self.topk_tokens, 0)
indices = torch.arange(
start,
start + self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
indices = indices.clamp(max=max_kv_len - 1).expand(num_tokens, -1)
self.topk_indices_buffer[:num_tokens] = indices
return
rows = []
offsets = torch.arange(
self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
) - (self.topk_tokens - 1)
for request in requests:
q_len = request.q_len
kv_len = request.kv_len
context_len = kv_len - q_len
positions = torch.arange(
context_len,
kv_len,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
row_indices = positions[:, None] + offsets[None, :]
rows.append(row_indices.clamp(min=0, max=kv_len - 1))
self.topk_indices_buffer[:num_tokens] = torch.cat(rows, dim=0)
return
raise ValueError(f"Unknown sparse MLA topk pattern: {pattern}")
class MockLayer(AttentionLayerBase):
"""Mock attention layer with scale parameters and impl.
@@ -252,10 +275,14 @@ class BenchmarkConfig:
num_kv_heads: int
block_size: int
device: str
max_model_len: int | None = None
dtype: torch.dtype = torch.float16
profile_memory: bool = False
use_cuda_graphs: bool = False
use_cuda_graphs: bool = True
ncu_profile: bool = False
torch_profile: bool = False
torch_profile_dir: str | None = None
torch_profile_iters: int = 3
warmup_ms: int | None = None
# "auto" or "fp8"
@@ -271,6 +298,10 @@ class BenchmarkConfig:
# Backend-specific tuning
num_kv_splits: int | None = None # CUTLASS MLA
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
sparse_mla_dense_mha_max_seq_len: int | None = None
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)