[1/N] Add dense MHA path for sparse MLA short sequences (#47327)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0b0ef8d7eb
commit
c4f5cd60da
@@ -4,8 +4,10 @@
|
||||
"""Common utilities for attention benchmarking."""
|
||||
|
||||
import csv
|
||||
import gc
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -44,10 +46,13 @@ def run_do_bench(
|
||||
kwargs: dict[str, Any] = {"return_mode": "all"}
|
||||
if use_cuda_graphs:
|
||||
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
if warmup_ms is not None:
|
||||
kwargs["warmup"] = warmup_ms
|
||||
result = triton.testing.do_bench(benchmark_fn, **kwargs)
|
||||
torch.accelerator.synchronize()
|
||||
return result
|
||||
|
||||
|
||||
@@ -91,42 +96,6 @@ except ImportError:
|
||||
AttentionLayerBase = object # Fallback
|
||||
|
||||
|
||||
class MockKVBProj:
|
||||
"""Mock KV projection layer for MLA prefill mode.
|
||||
|
||||
Mimics ColumnParallelLinear behavior for kv_b_proj in MLA backends.
|
||||
Projects kv_c_normed to [qk_nope_head_dim + v_head_dim] per head.
|
||||
"""
|
||||
|
||||
def __init__(self, num_heads: int, qk_nope_head_dim: int, v_head_dim: int):
|
||||
self.num_heads = num_heads
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.out_dim = qk_nope_head_dim + v_head_dim
|
||||
self.weight = torch.empty(0, dtype=torch.bfloat16)
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> tuple[torch.Tensor]:
|
||||
"""
|
||||
Project kv_c_normed to output space.
|
||||
|
||||
Args:
|
||||
x: Input tensor [num_tokens, kv_lora_rank]
|
||||
|
||||
Returns:
|
||||
Tuple containing output tensor
|
||||
[num_tokens, num_heads, qk_nope_head_dim + v_head_dim]
|
||||
"""
|
||||
num_tokens = x.shape[0]
|
||||
result = torch.randn(
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.out_dim,
|
||||
device=x.device,
|
||||
dtype=x.dtype,
|
||||
)
|
||||
return (result,) # Return as tuple to match ColumnParallelLinear API
|
||||
|
||||
|
||||
class MockIndexer:
|
||||
"""Mock Indexer for sparse MLA backends.
|
||||
|
||||
@@ -158,6 +127,60 @@ class MockIndexer:
|
||||
)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
|
||||
def fill_indices(
|
||||
self,
|
||||
num_tokens: int,
|
||||
max_kv_len: int,
|
||||
pattern: str = "random",
|
||||
requests: Sequence[Any] | None = None,
|
||||
):
|
||||
if pattern == "random":
|
||||
self.fill_random_indices(num_tokens, max_kv_len)
|
||||
return
|
||||
if pattern == "prefix":
|
||||
indices = torch.arange(
|
||||
self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
indices = (indices % max_kv_len).expand(num_tokens, -1)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
return
|
||||
if pattern == "sliding_window":
|
||||
if requests is None:
|
||||
start = max(max_kv_len - self.topk_tokens, 0)
|
||||
indices = torch.arange(
|
||||
start,
|
||||
start + self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
indices = indices.clamp(max=max_kv_len - 1).expand(num_tokens, -1)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
return
|
||||
|
||||
rows = []
|
||||
offsets = torch.arange(
|
||||
self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
) - (self.topk_tokens - 1)
|
||||
for request in requests:
|
||||
q_len = request.q_len
|
||||
kv_len = request.kv_len
|
||||
context_len = kv_len - q_len
|
||||
positions = torch.arange(
|
||||
context_len,
|
||||
kv_len,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
row_indices = positions[:, None] + offsets[None, :]
|
||||
rows.append(row_indices.clamp(min=0, max=kv_len - 1))
|
||||
self.topk_indices_buffer[:num_tokens] = torch.cat(rows, dim=0)
|
||||
return
|
||||
raise ValueError(f"Unknown sparse MLA topk pattern: {pattern}")
|
||||
|
||||
|
||||
class MockLayer(AttentionLayerBase):
|
||||
"""Mock attention layer with scale parameters and impl.
|
||||
@@ -252,10 +275,14 @@ class BenchmarkConfig:
|
||||
num_kv_heads: int
|
||||
block_size: int
|
||||
device: str
|
||||
max_model_len: int | None = None
|
||||
dtype: torch.dtype = torch.float16
|
||||
profile_memory: bool = False
|
||||
use_cuda_graphs: bool = False
|
||||
use_cuda_graphs: bool = True
|
||||
ncu_profile: bool = False
|
||||
torch_profile: bool = False
|
||||
torch_profile_dir: str | None = None
|
||||
torch_profile_iters: int = 3
|
||||
warmup_ms: int | None = None
|
||||
|
||||
# "auto" or "fp8"
|
||||
@@ -271,6 +298,10 @@ class BenchmarkConfig:
|
||||
# Backend-specific tuning
|
||||
num_kv_splits: int | None = None # CUTLASS MLA
|
||||
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
||||
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
|
||||
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
|
||||
sparse_mla_dense_mha_max_seq_len: int | None = None
|
||||
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
|
||||
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user