[1/N] Add dense MHA path for sparse MLA short sequences (#47327)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0b0ef8d7eb
commit
c4f5cd60da
@@ -75,7 +75,11 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
|
|||||||
from mla_runner import run_mla_benchmark as run_mla
|
from mla_runner import run_mla_benchmark as run_mla
|
||||||
|
|
||||||
return run_mla(
|
return run_mla(
|
||||||
config.backend, config, prefill_backend=config.prefill_backend, **kwargs
|
config.backend,
|
||||||
|
config,
|
||||||
|
prefill_backend=config.prefill_backend,
|
||||||
|
sparse_mla_force_mqa=config.sparse_mla_force_mqa,
|
||||||
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -592,6 +596,30 @@ def main():
|
|||||||
default="profile",
|
default="profile",
|
||||||
help="Output file name for ncu profile (default: 'profile').",
|
help="Output file name for ncu profile (default: 'profile').",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--torch-profile",
|
||||||
|
action="store_true",
|
||||||
|
default=False,
|
||||||
|
help="Collect a PyTorch profiler Chrome trace for each benchmark run.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--torch-profile-dir",
|
||||||
|
default=None,
|
||||||
|
help="Directory for PyTorch profiler traces.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--torch-profile-iters",
|
||||||
|
type=int,
|
||||||
|
default=3,
|
||||||
|
help="Number of forward passes to record per PyTorch profiler trace.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--sparse-mla-mha-variants",
|
||||||
|
nargs="+",
|
||||||
|
default=None,
|
||||||
|
choices=["dense_mha", "mqa"],
|
||||||
|
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
|
||||||
|
)
|
||||||
|
|
||||||
# Parameter sweep (use YAML config for advanced sweeps)
|
# Parameter sweep (use YAML config for advanced sweeps)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -641,6 +669,7 @@ def main():
|
|||||||
|
|
||||||
# Prefill backends (e.g., ["fa3", "fa4"])
|
# Prefill backends (e.g., ["fa3", "fa4"])
|
||||||
args.prefill_backends = yaml_config.get("prefill_backends", None)
|
args.prefill_backends = yaml_config.get("prefill_backends", None)
|
||||||
|
args.prefill_backend = yaml_config.get("prefill_backend", None)
|
||||||
|
|
||||||
# FP8 output benchmark knobs; CLI wins.
|
# FP8 output benchmark knobs; CLI wins.
|
||||||
if args.fp8_output_scale is None:
|
if args.fp8_output_scale is None:
|
||||||
@@ -683,6 +712,9 @@ def main():
|
|||||||
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
|
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
|
||||||
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
|
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
|
||||||
args.block_size = model.get("block_size", args.block_size)
|
args.block_size = model.get("block_size", args.block_size)
|
||||||
|
args.max_model_len = model.get(
|
||||||
|
"max_model_len", getattr(args, "max_model_len", None)
|
||||||
|
)
|
||||||
# MLA-specific dimensions
|
# MLA-specific dimensions
|
||||||
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
|
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
|
||||||
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
|
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
|
||||||
@@ -701,6 +733,21 @@ def main():
|
|||||||
args.cuda_graphs = yaml_config["cuda_graphs"]
|
args.cuda_graphs = yaml_config["cuda_graphs"]
|
||||||
if "ncu_profile" in yaml_config:
|
if "ncu_profile" in yaml_config:
|
||||||
args.ncu_profile = yaml_config["ncu_profile"]
|
args.ncu_profile = yaml_config["ncu_profile"]
|
||||||
|
if "torch_profile" in yaml_config:
|
||||||
|
args.torch_profile = yaml_config["torch_profile"]
|
||||||
|
if "torch_profile_dir" in yaml_config:
|
||||||
|
args.torch_profile_dir = yaml_config["torch_profile_dir"]
|
||||||
|
if "torch_profile_iters" in yaml_config:
|
||||||
|
args.torch_profile_iters = yaml_config["torch_profile_iters"]
|
||||||
|
args.sparse_mla_topk_pattern = yaml_config.get(
|
||||||
|
"sparse_mla_topk_pattern", "random"
|
||||||
|
)
|
||||||
|
args.sparse_mla_dense_mha_max_seq_len = yaml_config.get(
|
||||||
|
"sparse_mla_dense_mha_max_seq_len", None
|
||||||
|
)
|
||||||
|
args.sparse_mla_mha_variants = yaml_config.get(
|
||||||
|
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
|
||||||
|
)
|
||||||
|
|
||||||
# Parameter sweep configuration
|
# Parameter sweep configuration
|
||||||
if "parameter_sweep" in yaml_config:
|
if "parameter_sweep" in yaml_config:
|
||||||
@@ -842,8 +889,6 @@ def main():
|
|||||||
num_kv_heads=args.num_kv_heads,
|
num_kv_heads=args.num_kv_heads,
|
||||||
block_size=args.block_size,
|
block_size=args.block_size,
|
||||||
device=args.device,
|
device=args.device,
|
||||||
repeats=args.repeats,
|
|
||||||
warmup_iters=args.warmup_iters,
|
|
||||||
profile_memory=args.profile_memory,
|
profile_memory=args.profile_memory,
|
||||||
kv_cache_dtype=args.kv_cache_dtype,
|
kv_cache_dtype=args.kv_cache_dtype,
|
||||||
use_cuda_graphs=args.cuda_graphs,
|
use_cuda_graphs=args.cuda_graphs,
|
||||||
@@ -1063,6 +1108,133 @@ def main():
|
|||||||
f"\n [yellow]Prefill always faster for batch_size={bs}[/]"
|
f"\n [yellow]Prefill always faster for batch_size={bs}[/]"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Handle MHA vs MQA comparison mode for sparse MLA
|
||||||
|
elif hasattr(args, "mode") and args.mode == "mha_vs_mqa":
|
||||||
|
console.print("[yellow]Mode: MHA vs MQA comparison for sparse MLA[/]")
|
||||||
|
|
||||||
|
sparse_mla_topk_pattern = getattr(args, "sparse_mla_topk_pattern", "random")
|
||||||
|
dense_mha_max_seq_len = getattr(args, "sparse_mla_dense_mha_max_seq_len", None)
|
||||||
|
prefill_backend = getattr(args, "prefill_backend", None)
|
||||||
|
if prefill_backend:
|
||||||
|
console.print(f"Prefill backend: {prefill_backend}")
|
||||||
|
available_variants = [
|
||||||
|
("dense_mha", False, "dense"),
|
||||||
|
("mqa", True, "auto"),
|
||||||
|
]
|
||||||
|
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
|
||||||
|
if requested_variants is not None:
|
||||||
|
valid_variants = {label for label, _, _ in available_variants}
|
||||||
|
invalid_variants = sorted(set(requested_variants) - valid_variants)
|
||||||
|
if invalid_variants:
|
||||||
|
raise ValueError(
|
||||||
|
"Invalid sparse_mla_mha_variants entries: "
|
||||||
|
f"{invalid_variants}. Valid variants are: "
|
||||||
|
f"{sorted(valid_variants)}"
|
||||||
|
)
|
||||||
|
requested_variant_set = set(requested_variants)
|
||||||
|
variants = [
|
||||||
|
variant
|
||||||
|
for variant in available_variants
|
||||||
|
if variant[0] in requested_variant_set
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
variants = available_variants
|
||||||
|
formatter = ResultsFormatter(console)
|
||||||
|
total = 0
|
||||||
|
for spec in args.batch_specs:
|
||||||
|
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||||
|
for variant_label, _, _ in variants:
|
||||||
|
if (
|
||||||
|
variant_label == "dense_mha"
|
||||||
|
and dense_mha_max_seq_len is not None
|
||||||
|
and q_len > dense_mha_max_seq_len
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
total += len(backends)
|
||||||
|
|
||||||
|
with tqdm(total=total, desc="Benchmarking") as pbar:
|
||||||
|
for spec in args.batch_specs:
|
||||||
|
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||||
|
for backend in backends:
|
||||||
|
for variant_label, force_mqa, mha_mode in variants:
|
||||||
|
if (
|
||||||
|
variant_label == "dense_mha"
|
||||||
|
and dense_mha_max_seq_len is not None
|
||||||
|
and q_len > dense_mha_max_seq_len
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
config = BenchmarkConfig(
|
||||||
|
backend=f"{backend}_{variant_label}",
|
||||||
|
batch_spec=spec,
|
||||||
|
num_layers=args.num_layers,
|
||||||
|
head_dim=args.head_dim,
|
||||||
|
num_q_heads=args.num_q_heads,
|
||||||
|
num_kv_heads=args.num_kv_heads,
|
||||||
|
block_size=args.block_size,
|
||||||
|
device=args.device,
|
||||||
|
max_model_len=getattr(args, "max_model_len", None),
|
||||||
|
kv_cache_dtype=args.kv_cache_dtype,
|
||||||
|
profile_memory=args.profile_memory,
|
||||||
|
use_cuda_graphs=args.cuda_graphs,
|
||||||
|
ncu_profile=args.ncu_profile,
|
||||||
|
torch_profile=args.torch_profile,
|
||||||
|
torch_profile_dir=args.torch_profile_dir,
|
||||||
|
torch_profile_iters=args.torch_profile_iters,
|
||||||
|
warmup_ms=args.warmup_ms,
|
||||||
|
kv_lora_rank=getattr(args, "kv_lora_rank", None),
|
||||||
|
qk_nope_head_dim=getattr(args, "qk_nope_head_dim", None),
|
||||||
|
qk_rope_head_dim=getattr(args, "qk_rope_head_dim", None),
|
||||||
|
v_head_dim=getattr(args, "v_head_dim", None),
|
||||||
|
sparse_mla_force_mqa=force_mqa,
|
||||||
|
sparse_mla_mha_mode=mha_mode,
|
||||||
|
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
|
||||||
|
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
|
||||||
|
prefill_backend=prefill_backend,
|
||||||
|
)
|
||||||
|
|
||||||
|
# run_mla_benchmark needs the real backend name
|
||||||
|
from mla_runner import run_mla_benchmark as run_mla
|
||||||
|
|
||||||
|
run_label = f"{backend}_{variant_label} {spec}"
|
||||||
|
pbar.set_postfix_str(run_label)
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = run_mla(
|
||||||
|
backend,
|
||||||
|
config,
|
||||||
|
prefill_backend=prefill_backend,
|
||||||
|
sparse_mla_force_mqa=force_mqa,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
result = BenchmarkResult(
|
||||||
|
config=config,
|
||||||
|
mean_time=float("inf"),
|
||||||
|
median_time=float("inf"),
|
||||||
|
std_time=0,
|
||||||
|
min_time=float("inf"),
|
||||||
|
max_time=float("inf"),
|
||||||
|
error=str(e),
|
||||||
|
)
|
||||||
|
|
||||||
|
all_results.append(result)
|
||||||
|
if args.output_csv:
|
||||||
|
formatter.save_csv(all_results, args.output_csv)
|
||||||
|
if args.output_json:
|
||||||
|
formatter.save_json(all_results, args.output_json)
|
||||||
|
|
||||||
|
if not result.success:
|
||||||
|
console.print(
|
||||||
|
f"[red]Error {backend}_{variant_label} "
|
||||||
|
f"{spec}: {result.error}[/]"
|
||||||
|
)
|
||||||
|
|
||||||
|
pbar.update(1)
|
||||||
|
|
||||||
|
# Display results with variant labels as separate "backends"
|
||||||
|
console.print("\n[bold green]MHA vs MQA Results:[/]")
|
||||||
|
variant_backends = [f"{b}_{v}" for b in backends for v, _, _ in variants]
|
||||||
|
formatter.print_table(all_results, variant_backends)
|
||||||
|
|
||||||
# Handle model parameter sweep mode
|
# Handle model parameter sweep mode
|
||||||
elif hasattr(args, "model_parameter_sweep") and args.model_parameter_sweep:
|
elif hasattr(args, "model_parameter_sweep") and args.model_parameter_sweep:
|
||||||
# Model parameter sweep
|
# Model parameter sweep
|
||||||
|
|||||||
@@ -4,8 +4,10 @@
|
|||||||
"""Common utilities for attention benchmarking."""
|
"""Common utilities for attention benchmarking."""
|
||||||
|
|
||||||
import csv
|
import csv
|
||||||
|
import gc
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
|
from collections.abc import Sequence
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -44,10 +46,13 @@ def run_do_bench(
|
|||||||
kwargs: dict[str, Any] = {"return_mode": "all"}
|
kwargs: dict[str, Any] = {"return_mode": "all"}
|
||||||
if use_cuda_graphs:
|
if use_cuda_graphs:
|
||||||
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
|
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
|
||||||
|
gc.collect()
|
||||||
|
torch.accelerator.empty_cache()
|
||||||
else:
|
else:
|
||||||
if warmup_ms is not None:
|
if warmup_ms is not None:
|
||||||
kwargs["warmup"] = warmup_ms
|
kwargs["warmup"] = warmup_ms
|
||||||
result = triton.testing.do_bench(benchmark_fn, **kwargs)
|
result = triton.testing.do_bench(benchmark_fn, **kwargs)
|
||||||
|
torch.accelerator.synchronize()
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -91,42 +96,6 @@ except ImportError:
|
|||||||
AttentionLayerBase = object # Fallback
|
AttentionLayerBase = object # Fallback
|
||||||
|
|
||||||
|
|
||||||
class MockKVBProj:
|
|
||||||
"""Mock KV projection layer for MLA prefill mode.
|
|
||||||
|
|
||||||
Mimics ColumnParallelLinear behavior for kv_b_proj in MLA backends.
|
|
||||||
Projects kv_c_normed to [qk_nope_head_dim + v_head_dim] per head.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, num_heads: int, qk_nope_head_dim: int, v_head_dim: int):
|
|
||||||
self.num_heads = num_heads
|
|
||||||
self.qk_nope_head_dim = qk_nope_head_dim
|
|
||||||
self.v_head_dim = v_head_dim
|
|
||||||
self.out_dim = qk_nope_head_dim + v_head_dim
|
|
||||||
self.weight = torch.empty(0, dtype=torch.bfloat16)
|
|
||||||
|
|
||||||
def __call__(self, x: torch.Tensor) -> tuple[torch.Tensor]:
|
|
||||||
"""
|
|
||||||
Project kv_c_normed to output space.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
x: Input tensor [num_tokens, kv_lora_rank]
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple containing output tensor
|
|
||||||
[num_tokens, num_heads, qk_nope_head_dim + v_head_dim]
|
|
||||||
"""
|
|
||||||
num_tokens = x.shape[0]
|
|
||||||
result = torch.randn(
|
|
||||||
num_tokens,
|
|
||||||
self.num_heads,
|
|
||||||
self.out_dim,
|
|
||||||
device=x.device,
|
|
||||||
dtype=x.dtype,
|
|
||||||
)
|
|
||||||
return (result,) # Return as tuple to match ColumnParallelLinear API
|
|
||||||
|
|
||||||
|
|
||||||
class MockIndexer:
|
class MockIndexer:
|
||||||
"""Mock Indexer for sparse MLA backends.
|
"""Mock Indexer for sparse MLA backends.
|
||||||
|
|
||||||
@@ -158,6 +127,60 @@ class MockIndexer:
|
|||||||
)
|
)
|
||||||
self.topk_indices_buffer[:num_tokens] = indices
|
self.topk_indices_buffer[:num_tokens] = indices
|
||||||
|
|
||||||
|
def fill_indices(
|
||||||
|
self,
|
||||||
|
num_tokens: int,
|
||||||
|
max_kv_len: int,
|
||||||
|
pattern: str = "random",
|
||||||
|
requests: Sequence[Any] | None = None,
|
||||||
|
):
|
||||||
|
if pattern == "random":
|
||||||
|
self.fill_random_indices(num_tokens, max_kv_len)
|
||||||
|
return
|
||||||
|
if pattern == "prefix":
|
||||||
|
indices = torch.arange(
|
||||||
|
self.topk_tokens,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=self.topk_indices_buffer.device,
|
||||||
|
)
|
||||||
|
indices = (indices % max_kv_len).expand(num_tokens, -1)
|
||||||
|
self.topk_indices_buffer[:num_tokens] = indices
|
||||||
|
return
|
||||||
|
if pattern == "sliding_window":
|
||||||
|
if requests is None:
|
||||||
|
start = max(max_kv_len - self.topk_tokens, 0)
|
||||||
|
indices = torch.arange(
|
||||||
|
start,
|
||||||
|
start + self.topk_tokens,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=self.topk_indices_buffer.device,
|
||||||
|
)
|
||||||
|
indices = indices.clamp(max=max_kv_len - 1).expand(num_tokens, -1)
|
||||||
|
self.topk_indices_buffer[:num_tokens] = indices
|
||||||
|
return
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
offsets = torch.arange(
|
||||||
|
self.topk_tokens,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=self.topk_indices_buffer.device,
|
||||||
|
) - (self.topk_tokens - 1)
|
||||||
|
for request in requests:
|
||||||
|
q_len = request.q_len
|
||||||
|
kv_len = request.kv_len
|
||||||
|
context_len = kv_len - q_len
|
||||||
|
positions = torch.arange(
|
||||||
|
context_len,
|
||||||
|
kv_len,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=self.topk_indices_buffer.device,
|
||||||
|
)
|
||||||
|
row_indices = positions[:, None] + offsets[None, :]
|
||||||
|
rows.append(row_indices.clamp(min=0, max=kv_len - 1))
|
||||||
|
self.topk_indices_buffer[:num_tokens] = torch.cat(rows, dim=0)
|
||||||
|
return
|
||||||
|
raise ValueError(f"Unknown sparse MLA topk pattern: {pattern}")
|
||||||
|
|
||||||
|
|
||||||
class MockLayer(AttentionLayerBase):
|
class MockLayer(AttentionLayerBase):
|
||||||
"""Mock attention layer with scale parameters and impl.
|
"""Mock attention layer with scale parameters and impl.
|
||||||
@@ -252,10 +275,14 @@ class BenchmarkConfig:
|
|||||||
num_kv_heads: int
|
num_kv_heads: int
|
||||||
block_size: int
|
block_size: int
|
||||||
device: str
|
device: str
|
||||||
|
max_model_len: int | None = None
|
||||||
dtype: torch.dtype = torch.float16
|
dtype: torch.dtype = torch.float16
|
||||||
profile_memory: bool = False
|
profile_memory: bool = False
|
||||||
use_cuda_graphs: bool = False
|
use_cuda_graphs: bool = True
|
||||||
ncu_profile: bool = False
|
ncu_profile: bool = False
|
||||||
|
torch_profile: bool = False
|
||||||
|
torch_profile_dir: str | None = None
|
||||||
|
torch_profile_iters: int = 3
|
||||||
warmup_ms: int | None = None
|
warmup_ms: int | None = None
|
||||||
|
|
||||||
# "auto" or "fp8"
|
# "auto" or "fp8"
|
||||||
@@ -271,6 +298,10 @@ class BenchmarkConfig:
|
|||||||
# Backend-specific tuning
|
# Backend-specific tuning
|
||||||
num_kv_splits: int | None = None # CUTLASS MLA
|
num_kv_splits: int | None = None # CUTLASS MLA
|
||||||
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
||||||
|
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
|
||||||
|
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
|
||||||
|
sparse_mla_dense_mha_max_seq_len: int | None = None
|
||||||
|
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
|
||||||
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,474 @@
|
|||||||
|
# Sparse MLA benchmark: forward_mha vs forward_mqa
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
|
||||||
|
#
|
||||||
|
# Heatmap grid:
|
||||||
|
# - batch_size: 1, 2, 4, 8, 16, 32
|
||||||
|
# - seq_len: 32, 64, 128, 256, 512, 1024, 2048
|
||||||
|
# - q_len: powers of two through seq_len
|
||||||
|
#
|
||||||
|
# Specs with q_len < seq_len include context; the q_len == seq_len diagonal
|
||||||
|
# covers pure prefill.
|
||||||
|
# The model shape below is the DP case. For the TP8 run, manually change
|
||||||
|
# model.num_q_heads from 128 to 16 before rerunning this benchmark.
|
||||||
|
|
||||||
|
mode: mha_vs_mqa
|
||||||
|
|
||||||
|
model:
|
||||||
|
name: "deepseek-v3"
|
||||||
|
num_layers: 60
|
||||||
|
num_q_heads: 128
|
||||||
|
num_kv_heads: 1
|
||||||
|
head_dim: 576
|
||||||
|
kv_lora_rank: 512
|
||||||
|
qk_nope_head_dim: 128
|
||||||
|
qk_rope_head_dim: 64
|
||||||
|
v_head_dim: 128
|
||||||
|
block_size: 128
|
||||||
|
max_model_len: 2048
|
||||||
|
|
||||||
|
batch_specs:
|
||||||
|
# Batch size 1
|
||||||
|
# seq_len = 32
|
||||||
|
- "1q1s32"
|
||||||
|
- "1q2s32"
|
||||||
|
- "1q4s32"
|
||||||
|
- "1q8s32"
|
||||||
|
- "1q16s32"
|
||||||
|
- "1q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "1q1s64"
|
||||||
|
- "1q2s64"
|
||||||
|
- "1q4s64"
|
||||||
|
- "1q8s64"
|
||||||
|
- "1q16s64"
|
||||||
|
- "1q32s64"
|
||||||
|
- "1q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "1q1s128"
|
||||||
|
- "1q2s128"
|
||||||
|
- "1q4s128"
|
||||||
|
- "1q8s128"
|
||||||
|
- "1q16s128"
|
||||||
|
- "1q32s128"
|
||||||
|
- "1q64s128"
|
||||||
|
- "1q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "1q1s256"
|
||||||
|
- "1q2s256"
|
||||||
|
- "1q4s256"
|
||||||
|
- "1q8s256"
|
||||||
|
- "1q16s256"
|
||||||
|
- "1q32s256"
|
||||||
|
- "1q64s256"
|
||||||
|
- "1q128s256"
|
||||||
|
- "1q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "1q1s512"
|
||||||
|
- "1q2s512"
|
||||||
|
- "1q4s512"
|
||||||
|
- "1q8s512"
|
||||||
|
- "1q16s512"
|
||||||
|
- "1q32s512"
|
||||||
|
- "1q64s512"
|
||||||
|
- "1q128s512"
|
||||||
|
- "1q256s512"
|
||||||
|
- "1q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "1q1s1024"
|
||||||
|
- "1q2s1024"
|
||||||
|
- "1q4s1024"
|
||||||
|
- "1q8s1024"
|
||||||
|
- "1q16s1024"
|
||||||
|
- "1q32s1024"
|
||||||
|
- "1q64s1024"
|
||||||
|
- "1q128s1024"
|
||||||
|
- "1q256s1024"
|
||||||
|
- "1q512s1024"
|
||||||
|
- "1q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "1q1s2048"
|
||||||
|
- "1q2s2048"
|
||||||
|
- "1q4s2048"
|
||||||
|
- "1q8s2048"
|
||||||
|
- "1q16s2048"
|
||||||
|
- "1q32s2048"
|
||||||
|
- "1q64s2048"
|
||||||
|
- "1q128s2048"
|
||||||
|
- "1q256s2048"
|
||||||
|
- "1q512s2048"
|
||||||
|
- "1q1024s2048"
|
||||||
|
- "1q2048"
|
||||||
|
|
||||||
|
# Batch size 2
|
||||||
|
# seq_len = 32
|
||||||
|
- "2q1s32"
|
||||||
|
- "2q2s32"
|
||||||
|
- "2q4s32"
|
||||||
|
- "2q8s32"
|
||||||
|
- "2q16s32"
|
||||||
|
- "2q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "2q1s64"
|
||||||
|
- "2q2s64"
|
||||||
|
- "2q4s64"
|
||||||
|
- "2q8s64"
|
||||||
|
- "2q16s64"
|
||||||
|
- "2q32s64"
|
||||||
|
- "2q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "2q1s128"
|
||||||
|
- "2q2s128"
|
||||||
|
- "2q4s128"
|
||||||
|
- "2q8s128"
|
||||||
|
- "2q16s128"
|
||||||
|
- "2q32s128"
|
||||||
|
- "2q64s128"
|
||||||
|
- "2q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "2q1s256"
|
||||||
|
- "2q2s256"
|
||||||
|
- "2q4s256"
|
||||||
|
- "2q8s256"
|
||||||
|
- "2q16s256"
|
||||||
|
- "2q32s256"
|
||||||
|
- "2q64s256"
|
||||||
|
- "2q128s256"
|
||||||
|
- "2q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "2q1s512"
|
||||||
|
- "2q2s512"
|
||||||
|
- "2q4s512"
|
||||||
|
- "2q8s512"
|
||||||
|
- "2q16s512"
|
||||||
|
- "2q32s512"
|
||||||
|
- "2q64s512"
|
||||||
|
- "2q128s512"
|
||||||
|
- "2q256s512"
|
||||||
|
- "2q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "2q1s1024"
|
||||||
|
- "2q2s1024"
|
||||||
|
- "2q4s1024"
|
||||||
|
- "2q8s1024"
|
||||||
|
- "2q16s1024"
|
||||||
|
- "2q32s1024"
|
||||||
|
- "2q64s1024"
|
||||||
|
- "2q128s1024"
|
||||||
|
- "2q256s1024"
|
||||||
|
- "2q512s1024"
|
||||||
|
- "2q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "2q1s2048"
|
||||||
|
- "2q2s2048"
|
||||||
|
- "2q4s2048"
|
||||||
|
- "2q8s2048"
|
||||||
|
- "2q16s2048"
|
||||||
|
- "2q32s2048"
|
||||||
|
- "2q64s2048"
|
||||||
|
- "2q128s2048"
|
||||||
|
- "2q256s2048"
|
||||||
|
- "2q512s2048"
|
||||||
|
- "2q1024s2048"
|
||||||
|
- "2q2048"
|
||||||
|
|
||||||
|
# Batch size 4
|
||||||
|
# seq_len = 32
|
||||||
|
- "4q1s32"
|
||||||
|
- "4q2s32"
|
||||||
|
- "4q4s32"
|
||||||
|
- "4q8s32"
|
||||||
|
- "4q16s32"
|
||||||
|
- "4q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "4q1s64"
|
||||||
|
- "4q2s64"
|
||||||
|
- "4q4s64"
|
||||||
|
- "4q8s64"
|
||||||
|
- "4q16s64"
|
||||||
|
- "4q32s64"
|
||||||
|
- "4q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "4q1s128"
|
||||||
|
- "4q2s128"
|
||||||
|
- "4q4s128"
|
||||||
|
- "4q8s128"
|
||||||
|
- "4q16s128"
|
||||||
|
- "4q32s128"
|
||||||
|
- "4q64s128"
|
||||||
|
- "4q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "4q1s256"
|
||||||
|
- "4q2s256"
|
||||||
|
- "4q4s256"
|
||||||
|
- "4q8s256"
|
||||||
|
- "4q16s256"
|
||||||
|
- "4q32s256"
|
||||||
|
- "4q64s256"
|
||||||
|
- "4q128s256"
|
||||||
|
- "4q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "4q1s512"
|
||||||
|
- "4q2s512"
|
||||||
|
- "4q4s512"
|
||||||
|
- "4q8s512"
|
||||||
|
- "4q16s512"
|
||||||
|
- "4q32s512"
|
||||||
|
- "4q64s512"
|
||||||
|
- "4q128s512"
|
||||||
|
- "4q256s512"
|
||||||
|
- "4q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "4q1s1024"
|
||||||
|
- "4q2s1024"
|
||||||
|
- "4q4s1024"
|
||||||
|
- "4q8s1024"
|
||||||
|
- "4q16s1024"
|
||||||
|
- "4q32s1024"
|
||||||
|
- "4q64s1024"
|
||||||
|
- "4q128s1024"
|
||||||
|
- "4q256s1024"
|
||||||
|
- "4q512s1024"
|
||||||
|
- "4q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "4q1s2048"
|
||||||
|
- "4q2s2048"
|
||||||
|
- "4q4s2048"
|
||||||
|
- "4q8s2048"
|
||||||
|
- "4q16s2048"
|
||||||
|
- "4q32s2048"
|
||||||
|
- "4q64s2048"
|
||||||
|
- "4q128s2048"
|
||||||
|
- "4q256s2048"
|
||||||
|
- "4q512s2048"
|
||||||
|
- "4q1024s2048"
|
||||||
|
- "4q2048"
|
||||||
|
|
||||||
|
# Batch size 8
|
||||||
|
# seq_len = 32
|
||||||
|
- "8q1s32"
|
||||||
|
- "8q2s32"
|
||||||
|
- "8q4s32"
|
||||||
|
- "8q8s32"
|
||||||
|
- "8q16s32"
|
||||||
|
- "8q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "8q1s64"
|
||||||
|
- "8q2s64"
|
||||||
|
- "8q4s64"
|
||||||
|
- "8q8s64"
|
||||||
|
- "8q16s64"
|
||||||
|
- "8q32s64"
|
||||||
|
- "8q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "8q1s128"
|
||||||
|
- "8q2s128"
|
||||||
|
- "8q4s128"
|
||||||
|
- "8q8s128"
|
||||||
|
- "8q16s128"
|
||||||
|
- "8q32s128"
|
||||||
|
- "8q64s128"
|
||||||
|
- "8q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "8q1s256"
|
||||||
|
- "8q2s256"
|
||||||
|
- "8q4s256"
|
||||||
|
- "8q8s256"
|
||||||
|
- "8q16s256"
|
||||||
|
- "8q32s256"
|
||||||
|
- "8q64s256"
|
||||||
|
- "8q128s256"
|
||||||
|
- "8q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "8q1s512"
|
||||||
|
- "8q2s512"
|
||||||
|
- "8q4s512"
|
||||||
|
- "8q8s512"
|
||||||
|
- "8q16s512"
|
||||||
|
- "8q32s512"
|
||||||
|
- "8q64s512"
|
||||||
|
- "8q128s512"
|
||||||
|
- "8q256s512"
|
||||||
|
- "8q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "8q1s1024"
|
||||||
|
- "8q2s1024"
|
||||||
|
- "8q4s1024"
|
||||||
|
- "8q8s1024"
|
||||||
|
- "8q16s1024"
|
||||||
|
- "8q32s1024"
|
||||||
|
- "8q64s1024"
|
||||||
|
- "8q128s1024"
|
||||||
|
- "8q256s1024"
|
||||||
|
- "8q512s1024"
|
||||||
|
- "8q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "8q1s2048"
|
||||||
|
- "8q2s2048"
|
||||||
|
- "8q4s2048"
|
||||||
|
- "8q8s2048"
|
||||||
|
- "8q16s2048"
|
||||||
|
- "8q32s2048"
|
||||||
|
- "8q64s2048"
|
||||||
|
- "8q128s2048"
|
||||||
|
- "8q256s2048"
|
||||||
|
- "8q512s2048"
|
||||||
|
- "8q1024s2048"
|
||||||
|
- "8q2048"
|
||||||
|
|
||||||
|
# Batch size 16
|
||||||
|
# seq_len = 32
|
||||||
|
- "16q1s32"
|
||||||
|
- "16q2s32"
|
||||||
|
- "16q4s32"
|
||||||
|
- "16q8s32"
|
||||||
|
- "16q16s32"
|
||||||
|
- "16q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "16q1s64"
|
||||||
|
- "16q2s64"
|
||||||
|
- "16q4s64"
|
||||||
|
- "16q8s64"
|
||||||
|
- "16q16s64"
|
||||||
|
- "16q32s64"
|
||||||
|
- "16q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "16q1s128"
|
||||||
|
- "16q2s128"
|
||||||
|
- "16q4s128"
|
||||||
|
- "16q8s128"
|
||||||
|
- "16q16s128"
|
||||||
|
- "16q32s128"
|
||||||
|
- "16q64s128"
|
||||||
|
- "16q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "16q1s256"
|
||||||
|
- "16q2s256"
|
||||||
|
- "16q4s256"
|
||||||
|
- "16q8s256"
|
||||||
|
- "16q16s256"
|
||||||
|
- "16q32s256"
|
||||||
|
- "16q64s256"
|
||||||
|
- "16q128s256"
|
||||||
|
- "16q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "16q1s512"
|
||||||
|
- "16q2s512"
|
||||||
|
- "16q4s512"
|
||||||
|
- "16q8s512"
|
||||||
|
- "16q16s512"
|
||||||
|
- "16q32s512"
|
||||||
|
- "16q64s512"
|
||||||
|
- "16q128s512"
|
||||||
|
- "16q256s512"
|
||||||
|
- "16q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "16q1s1024"
|
||||||
|
- "16q2s1024"
|
||||||
|
- "16q4s1024"
|
||||||
|
- "16q8s1024"
|
||||||
|
- "16q16s1024"
|
||||||
|
- "16q32s1024"
|
||||||
|
- "16q64s1024"
|
||||||
|
- "16q128s1024"
|
||||||
|
- "16q256s1024"
|
||||||
|
- "16q512s1024"
|
||||||
|
- "16q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "16q1s2048"
|
||||||
|
- "16q2s2048"
|
||||||
|
- "16q4s2048"
|
||||||
|
- "16q8s2048"
|
||||||
|
- "16q16s2048"
|
||||||
|
- "16q32s2048"
|
||||||
|
- "16q64s2048"
|
||||||
|
- "16q128s2048"
|
||||||
|
- "16q256s2048"
|
||||||
|
- "16q512s2048"
|
||||||
|
- "16q1024s2048"
|
||||||
|
- "16q2048"
|
||||||
|
|
||||||
|
# Batch size 32
|
||||||
|
# seq_len = 32
|
||||||
|
- "32q1s32"
|
||||||
|
- "32q2s32"
|
||||||
|
- "32q4s32"
|
||||||
|
- "32q8s32"
|
||||||
|
- "32q16s32"
|
||||||
|
- "32q32"
|
||||||
|
# seq_len = 64
|
||||||
|
- "32q1s64"
|
||||||
|
- "32q2s64"
|
||||||
|
- "32q4s64"
|
||||||
|
- "32q8s64"
|
||||||
|
- "32q16s64"
|
||||||
|
- "32q32s64"
|
||||||
|
- "32q64"
|
||||||
|
# seq_len = 128
|
||||||
|
- "32q1s128"
|
||||||
|
- "32q2s128"
|
||||||
|
- "32q4s128"
|
||||||
|
- "32q8s128"
|
||||||
|
- "32q16s128"
|
||||||
|
- "32q32s128"
|
||||||
|
- "32q64s128"
|
||||||
|
- "32q128"
|
||||||
|
# seq_len = 256
|
||||||
|
- "32q1s256"
|
||||||
|
- "32q2s256"
|
||||||
|
- "32q4s256"
|
||||||
|
- "32q8s256"
|
||||||
|
- "32q16s256"
|
||||||
|
- "32q32s256"
|
||||||
|
- "32q64s256"
|
||||||
|
- "32q128s256"
|
||||||
|
- "32q256"
|
||||||
|
# seq_len = 512
|
||||||
|
- "32q1s512"
|
||||||
|
- "32q2s512"
|
||||||
|
- "32q4s512"
|
||||||
|
- "32q8s512"
|
||||||
|
- "32q16s512"
|
||||||
|
- "32q32s512"
|
||||||
|
- "32q64s512"
|
||||||
|
- "32q128s512"
|
||||||
|
- "32q256s512"
|
||||||
|
- "32q512"
|
||||||
|
# seq_len = 1024
|
||||||
|
- "32q1s1024"
|
||||||
|
- "32q2s1024"
|
||||||
|
- "32q4s1024"
|
||||||
|
- "32q8s1024"
|
||||||
|
- "32q16s1024"
|
||||||
|
- "32q32s1024"
|
||||||
|
- "32q64s1024"
|
||||||
|
- "32q128s1024"
|
||||||
|
- "32q256s1024"
|
||||||
|
- "32q512s1024"
|
||||||
|
- "32q1024"
|
||||||
|
# seq_len = 2048
|
||||||
|
- "32q1s2048"
|
||||||
|
- "32q2s2048"
|
||||||
|
- "32q4s2048"
|
||||||
|
- "32q8s2048"
|
||||||
|
- "32q16s2048"
|
||||||
|
- "32q32s2048"
|
||||||
|
- "32q64s2048"
|
||||||
|
- "32q128s2048"
|
||||||
|
- "32q256s2048"
|
||||||
|
- "32q512s2048"
|
||||||
|
- "32q1024s2048"
|
||||||
|
- "32q2048"
|
||||||
|
|
||||||
|
backends:
|
||||||
|
- FLASHMLA_SPARSE
|
||||||
|
|
||||||
|
device: "cuda:0"
|
||||||
|
profile_memory: false
|
||||||
|
sparse_mla_dense_mha_max_seq_len: 2048
|
||||||
|
sparse_mla_topk_pattern: "random"
|
||||||
|
|
||||||
|
output:
|
||||||
|
csv: "benchmark_output/mla_sparse_mha_vs_mqa.csv"
|
||||||
|
json: "benchmark_output/mla_sparse_mha_vs_mqa.json"
|
||||||
@@ -9,6 +9,8 @@ needing full VllmConfig integration.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import statistics
|
import statistics
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -17,7 +19,6 @@ from common import (
|
|||||||
BenchmarkResult,
|
BenchmarkResult,
|
||||||
MockHfConfig,
|
MockHfConfig,
|
||||||
MockIndexer,
|
MockIndexer,
|
||||||
MockKVBProj,
|
|
||||||
MockLayer,
|
MockLayer,
|
||||||
run_do_bench,
|
run_do_bench,
|
||||||
run_ncu_profile,
|
run_ncu_profile,
|
||||||
@@ -33,8 +34,59 @@ from vllm.config import (
|
|||||||
VllmConfig,
|
VllmConfig,
|
||||||
set_current_vllm_config,
|
set_current_vllm_config,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_profile_name(value: str) -> str:
|
||||||
|
return "".join(c if c.isalnum() or c in "._-" else "_" for c in value)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_kv_b_proj(
|
||||||
|
mla_dims: dict,
|
||||||
|
device: torch.device,
|
||||||
|
):
|
||||||
|
kv_b_proj = ColumnParallelLinear(
|
||||||
|
mla_dims["kv_lora_rank"],
|
||||||
|
mla_dims["num_q_heads"]
|
||||||
|
* (mla_dims["qk_nope_head_dim"] + mla_dims["v_head_dim"]),
|
||||||
|
bias=False,
|
||||||
|
params_dtype=torch.bfloat16,
|
||||||
|
quant_config=None,
|
||||||
|
prefix="benchmark.kv_b_proj",
|
||||||
|
).to(device)
|
||||||
|
with torch.no_grad():
|
||||||
|
kv_b_proj.weight.copy_(torch.randn_like(kv_b_proj.weight))
|
||||||
|
return kv_b_proj
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_single_rank_model_parallel() -> None:
|
||||||
|
import torch.distributed as dist
|
||||||
|
|
||||||
|
from vllm.distributed import (
|
||||||
|
ensure_model_parallel_initialized,
|
||||||
|
init_distributed_environment,
|
||||||
|
model_parallel_is_initialized,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not dist.is_available():
|
||||||
|
return
|
||||||
|
if not dist.is_initialized():
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
prefix="vllm_bench_dist_", delete=False
|
||||||
|
) as init_file:
|
||||||
|
distributed_init_method = f"file://{init_file.name}"
|
||||||
|
init_distributed_environment(
|
||||||
|
world_size=1,
|
||||||
|
rank=0,
|
||||||
|
distributed_init_method=distributed_init_method,
|
||||||
|
local_rank=0,
|
||||||
|
backend="nccl",
|
||||||
|
)
|
||||||
|
if not model_parallel_is_initialized():
|
||||||
|
ensure_model_parallel_initialized(1, 1)
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# VllmConfig Creation
|
# VllmConfig Creation
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
@@ -66,10 +118,12 @@ def create_minimal_vllm_config(
|
|||||||
block_size: int = 128,
|
block_size: int = 128,
|
||||||
max_num_seqs: int = 256,
|
max_num_seqs: int = 256,
|
||||||
max_num_batched_tokens: int = 8192,
|
max_num_batched_tokens: int = 8192,
|
||||||
|
max_model_len: int = 32768,
|
||||||
mla_dims: dict | None = None,
|
mla_dims: dict | None = None,
|
||||||
index_topk: int | None = None,
|
index_topk: int | None = None,
|
||||||
prefill_backend: str | None = None,
|
prefill_backend: str | None = None,
|
||||||
kv_cache_dtype: str = "auto",
|
kv_cache_dtype: str = "auto",
|
||||||
|
sparse_mla_force_mqa: bool = False,
|
||||||
) -> VllmConfig:
|
) -> VllmConfig:
|
||||||
"""
|
"""
|
||||||
Create minimal VllmConfig for MLA benchmarks.
|
Create minimal VllmConfig for MLA benchmarks.
|
||||||
@@ -86,6 +140,8 @@ def create_minimal_vllm_config(
|
|||||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4", "flashinfer",
|
prefill_backend: Prefill backend name (e.g., "fa3", "fa4", "flashinfer",
|
||||||
"trtllm"). Configures the attention config to force
|
"trtllm"). Configures the attention config to force
|
||||||
the specified prefill backend.
|
the specified prefill backend.
|
||||||
|
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||||
|
forward_mqa (even prefill tokens).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
VllmConfig for benchmarking
|
VllmConfig for benchmarking
|
||||||
@@ -131,7 +187,7 @@ def create_minimal_vllm_config(
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
dtype="bfloat16",
|
dtype="bfloat16",
|
||||||
seed=0,
|
seed=0,
|
||||||
max_model_len=32768,
|
max_model_len=max_model_len,
|
||||||
quantization=None,
|
quantization=None,
|
||||||
enforce_eager=False,
|
enforce_eager=False,
|
||||||
max_logprobs=20,
|
max_logprobs=20,
|
||||||
@@ -163,7 +219,7 @@ def create_minimal_vllm_config(
|
|||||||
scheduler_config = SchedulerConfig(
|
scheduler_config = SchedulerConfig(
|
||||||
max_num_seqs=max_num_seqs,
|
max_num_seqs=max_num_seqs,
|
||||||
max_num_batched_tokens=max(max_num_batched_tokens, max_num_seqs),
|
max_num_batched_tokens=max(max_num_batched_tokens, max_num_seqs),
|
||||||
max_model_len=32768,
|
max_model_len=max_model_len,
|
||||||
is_encoder_decoder=False,
|
is_encoder_decoder=False,
|
||||||
enable_chunked_prefill=True,
|
enable_chunked_prefill=True,
|
||||||
)
|
)
|
||||||
@@ -192,6 +248,9 @@ def create_minimal_vllm_config(
|
|||||||
"flash_attn_version"
|
"flash_attn_version"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if sparse_mla_force_mqa:
|
||||||
|
vllm_config.attention_config.sparse_mla_force_mqa = True
|
||||||
|
|
||||||
return vllm_config
|
return vllm_config
|
||||||
|
|
||||||
|
|
||||||
@@ -548,12 +607,7 @@ def _create_backend_impl(
|
|||||||
# Calculate scale
|
# Calculate scale
|
||||||
scale = 1.0 / np.sqrt(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"])
|
scale = 1.0 / np.sqrt(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"])
|
||||||
|
|
||||||
# Create mock kv_b_proj layer for prefill mode
|
kv_b_proj = _create_kv_b_proj(mla_dims, device)
|
||||||
mock_kv_b_proj = MockKVBProj(
|
|
||||||
num_heads=mla_dims["num_q_heads"],
|
|
||||||
qk_nope_head_dim=mla_dims["qk_nope_head_dim"],
|
|
||||||
v_head_dim=mla_dims["v_head_dim"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create indexer for sparse backends
|
# Create indexer for sparse backends
|
||||||
indexer = None
|
indexer = None
|
||||||
@@ -584,7 +638,7 @@ def _create_backend_impl(
|
|||||||
"qk_rope_head_dim": mla_dims["qk_rope_head_dim"],
|
"qk_rope_head_dim": mla_dims["qk_rope_head_dim"],
|
||||||
"qk_head_dim": mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"],
|
"qk_head_dim": mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"],
|
||||||
"v_head_dim": mla_dims["v_head_dim"],
|
"v_head_dim": mla_dims["v_head_dim"],
|
||||||
"kv_b_proj": mock_kv_b_proj,
|
"kv_b_proj": kv_b_proj,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Add indexer for sparse backends
|
# Add indexer for sparse backends
|
||||||
@@ -785,14 +839,35 @@ def _run_single_benchmark(
|
|||||||
# Fill indexer with random indices for sparse backends
|
# Fill indexer with random indices for sparse backends
|
||||||
is_sparse = backend_cfg.get("is_sparse", False)
|
is_sparse = backend_cfg.get("is_sparse", False)
|
||||||
if is_sparse and indexer is not None:
|
if is_sparse and indexer is not None:
|
||||||
indexer.fill_random_indices(total_q, max_kv_len)
|
indexer.fill_indices(
|
||||||
|
total_q,
|
||||||
|
max_kv_len,
|
||||||
|
getattr(config, "sparse_mla_topk_pattern", "random"),
|
||||||
|
)
|
||||||
|
|
||||||
# Determine which forward methods to use based on metadata.
|
# Determine which forward methods to use based on metadata.
|
||||||
# Sparse MLA backends always use forward_mqa
|
# Non-sparse backends use .decode/.prefill sub-objects.
|
||||||
has_decode = is_sparse or getattr(metadata, "decode", None) is not None
|
# Sparse backends use num_decode_tokens/num_prefills directly.
|
||||||
has_prefill = not is_sparse and getattr(metadata, "prefill", None) is not None
|
#
|
||||||
|
# sparse_mla_force_mqa overrides: even for prefill metadata, use MQA.
|
||||||
|
force_mqa = getattr(config, "sparse_mla_force_mqa", False)
|
||||||
|
force_dense_mha = getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
|
||||||
|
if force_mqa:
|
||||||
|
has_decode = True
|
||||||
|
has_prefill = False
|
||||||
|
elif is_sparse:
|
||||||
|
has_decode = metadata.num_decode_tokens > 0
|
||||||
|
has_prefill = metadata.num_prefills > 0
|
||||||
|
else:
|
||||||
|
has_decode = metadata.decode is not None
|
||||||
|
has_prefill = metadata.prefill is not None
|
||||||
if not has_decode and not has_prefill:
|
if not has_decode and not has_prefill:
|
||||||
raise RuntimeError("Metadata has neither decode nor prefill metadata")
|
raise RuntimeError("Metadata has neither decode nor prefill metadata")
|
||||||
|
if is_sparse and force_dense_mha and not has_prefill:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Sparse MLA dense_mha benchmark did not produce prefill metadata. "
|
||||||
|
"Check reorder_batch_threshold/path forcing."
|
||||||
|
)
|
||||||
|
|
||||||
num_decode = (
|
num_decode = (
|
||||||
metadata.num_decode_tokens
|
metadata.num_decode_tokens
|
||||||
@@ -871,7 +946,6 @@ def _run_single_benchmark(
|
|||||||
metadata,
|
metadata,
|
||||||
prefill_inputs["k_scale"],
|
prefill_inputs["k_scale"],
|
||||||
prefill_fp8_output if fused_output else prefill_inputs["output"],
|
prefill_fp8_output if fused_output else prefill_inputs["output"],
|
||||||
prefill_output_scale if fused_output else None,
|
|
||||||
)
|
)
|
||||||
if fused_output:
|
if fused_output:
|
||||||
out = prefill_fp8_output
|
out = prefill_fp8_output
|
||||||
@@ -898,6 +972,48 @@ def _run_single_benchmark(
|
|||||||
throughput_tokens_per_sec=0.0,
|
throughput_tokens_per_sec=0.0,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if config.torch_profile:
|
||||||
|
profile_dir = Path(
|
||||||
|
config.torch_profile_dir or "benchmark_outputs/torch_profiles"
|
||||||
|
)
|
||||||
|
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
trace_name = _safe_profile_name(f"{config.backend}_{config.batch_spec}")
|
||||||
|
trace_path = profile_dir / f"{trace_name}.json"
|
||||||
|
iters = max(config.torch_profile_iters, 1)
|
||||||
|
|
||||||
|
forward_fn()
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
with torch.profiler.profile(
|
||||||
|
activities=[
|
||||||
|
torch.profiler.ProfilerActivity.CPU,
|
||||||
|
torch.profiler.ProfilerActivity.CUDA,
|
||||||
|
],
|
||||||
|
record_shapes=True,
|
||||||
|
profile_memory=True,
|
||||||
|
with_stack=False,
|
||||||
|
) as prof:
|
||||||
|
for _ in range(iters):
|
||||||
|
forward_fn()
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
prof.step()
|
||||||
|
prof.export_chrome_trace(str(trace_path))
|
||||||
|
print(f"Saved PyTorch profiler trace to {trace_path}")
|
||||||
|
print(
|
||||||
|
prof.key_averages().table(
|
||||||
|
sort_by="cuda_time_total",
|
||||||
|
row_limit=25,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return BenchmarkResult(
|
||||||
|
config=config,
|
||||||
|
mean_time=0.0,
|
||||||
|
median_time=0.0,
|
||||||
|
std_time=0.0,
|
||||||
|
min_time=0.0,
|
||||||
|
max_time=0.0,
|
||||||
|
throughput_tokens_per_sec=0.0,
|
||||||
|
)
|
||||||
|
|
||||||
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
|
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
|
||||||
|
|
||||||
# Convert ms to seconds per layer
|
# Convert ms to seconds per layer
|
||||||
@@ -920,6 +1036,7 @@ def _run_mla_benchmark_batched(
|
|||||||
configs_with_params: list[tuple], # [(config, threshold, num_splits), ...]
|
configs_with_params: list[tuple], # [(config, threshold, num_splits), ...]
|
||||||
index_topk: int = 2048,
|
index_topk: int = 2048,
|
||||||
prefill_backend: str | None = None,
|
prefill_backend: str | None = None,
|
||||||
|
sparse_mla_force_mqa: bool = False,
|
||||||
output_scale: float | None = None,
|
output_scale: float | None = None,
|
||||||
fuse_quant_op: bool = False,
|
fuse_quant_op: bool = False,
|
||||||
) -> list[BenchmarkResult]:
|
) -> list[BenchmarkResult]:
|
||||||
@@ -940,6 +1057,8 @@ def _run_mla_benchmark_batched(
|
|||||||
index_topk: Topk value for sparse MLA backends (default 2048)
|
index_topk: Topk value for sparse MLA backends (default 2048)
|
||||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
||||||
When set, forces the specified FlashAttention version for prefill.
|
When set, forces the specified FlashAttention version for prefill.
|
||||||
|
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||||
|
forward_mqa (even prefill tokens).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of BenchmarkResult objects
|
List of BenchmarkResult objects
|
||||||
@@ -980,21 +1099,41 @@ def _run_mla_benchmark_batched(
|
|||||||
sum(r.q_len for r in parse_batch_spec(cfg.batch_spec))
|
sum(r.q_len for r in parse_batch_spec(cfg.batch_spec))
|
||||||
for cfg, *_ in configs_with_params
|
for cfg, *_ in configs_with_params
|
||||||
)
|
)
|
||||||
|
max_model_len = max(
|
||||||
|
max_total_q,
|
||||||
|
max(
|
||||||
|
getattr(cfg, "max_model_len", None) or 32768
|
||||||
|
for cfg, *_ in configs_with_params
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
# Create and set vLLM config for MLA (reused across all benchmarks)
|
# Create and set vLLM config for MLA (reused across all benchmarks)
|
||||||
vllm_config = create_minimal_vllm_config(
|
vllm_config = create_minimal_vllm_config(
|
||||||
model_name="deepseek-v3", # Used only for model path
|
model_name="deepseek-v3", # Used only for model path
|
||||||
block_size=block_size,
|
block_size=block_size,
|
||||||
max_num_batched_tokens=max_total_q,
|
max_num_batched_tokens=max_total_q,
|
||||||
|
max_model_len=max_model_len,
|
||||||
mla_dims=mla_dims, # Use custom dims from config or default
|
mla_dims=mla_dims, # Use custom dims from config or default
|
||||||
index_topk=index_topk if is_sparse else None,
|
index_topk=index_topk if is_sparse else None,
|
||||||
prefill_backend=prefill_backend,
|
prefill_backend=prefill_backend,
|
||||||
kv_cache_dtype=kv_cache_dtype,
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
sparse_mla_force_mqa=sparse_mla_force_mqa,
|
||||||
)
|
)
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
|
|
||||||
|
# Initialize workspace manager (needed by metadata builders)
|
||||||
|
from vllm.v1.worker.workspace import (
|
||||||
|
init_workspace_manager,
|
||||||
|
is_workspace_manager_initialized,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not is_workspace_manager_initialized():
|
||||||
|
init_workspace_manager(device)
|
||||||
|
|
||||||
with set_current_vllm_config(vllm_config):
|
with set_current_vllm_config(vllm_config):
|
||||||
|
_ensure_single_rank_model_parallel()
|
||||||
|
|
||||||
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
|
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
|
||||||
impl, layer, builder_instance, indexer = _create_backend_impl(
|
impl, layer, builder_instance, indexer = _create_backend_impl(
|
||||||
backend_cfg,
|
backend_cfg,
|
||||||
@@ -1040,9 +1179,20 @@ def _run_mla_benchmark_batched(
|
|||||||
for config, threshold, num_splits in configs_with_params:
|
for config, threshold, num_splits in configs_with_params:
|
||||||
# Set threshold for this benchmark (FlashAttn/FlashMLA only)
|
# Set threshold for this benchmark (FlashAttn/FlashMLA only)
|
||||||
original_threshold = None
|
original_threshold = None
|
||||||
if threshold is not None and builder_instance:
|
effective_threshold = threshold
|
||||||
|
force_dense_mha = (
|
||||||
|
is_sparse
|
||||||
|
and getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
|
||||||
|
and not getattr(config, "sparse_mla_force_mqa", False)
|
||||||
|
)
|
||||||
|
if force_dense_mha:
|
||||||
|
# Sparse MLA normally treats q_len <= 1 as decode. Use an
|
||||||
|
# impossible threshold so dense_mha benchmarks actually run
|
||||||
|
# the prefill/MHA path, including q_len=1 short extends.
|
||||||
|
effective_threshold = -1
|
||||||
|
if effective_threshold is not None and builder_instance:
|
||||||
original_threshold = builder_instance.reorder_batch_threshold
|
original_threshold = builder_instance.reorder_batch_threshold
|
||||||
builder_instance.reorder_batch_threshold = threshold
|
builder_instance.reorder_batch_threshold = effective_threshold
|
||||||
|
|
||||||
# Set num_splits for CUTLASS
|
# Set num_splits for CUTLASS
|
||||||
original_num_splits = None
|
original_num_splits = None
|
||||||
@@ -1090,6 +1240,7 @@ def run_mla_benchmark(
|
|||||||
num_kv_splits: int | None = None,
|
num_kv_splits: int | None = None,
|
||||||
index_topk: int = 2048,
|
index_topk: int = 2048,
|
||||||
prefill_backend: str | None = None,
|
prefill_backend: str | None = None,
|
||||||
|
sparse_mla_force_mqa: bool = False,
|
||||||
output_scale: float | None = None,
|
output_scale: float | None = None,
|
||||||
fuse_quant_op: bool = False,
|
fuse_quant_op: bool = False,
|
||||||
) -> BenchmarkResult | list[BenchmarkResult]:
|
) -> BenchmarkResult | list[BenchmarkResult]:
|
||||||
@@ -1111,6 +1262,8 @@ def run_mla_benchmark(
|
|||||||
index_topk: Topk value for sparse MLA backends (default 2048)
|
index_topk: Topk value for sparse MLA backends (default 2048)
|
||||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
||||||
When set, forces the specified FlashAttention version for prefill.
|
When set, forces the specified FlashAttention version for prefill.
|
||||||
|
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||||
|
forward_mqa (even prefill tokens).
|
||||||
output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
|
output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
|
||||||
fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
|
fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
|
||||||
kernel vs a standalone post-quant kernel. See _run_single_benchmark.
|
kernel vs a standalone post-quant kernel. See _run_single_benchmark.
|
||||||
@@ -1142,6 +1295,7 @@ def run_mla_benchmark(
|
|||||||
configs_with_params,
|
configs_with_params,
|
||||||
index_topk,
|
index_topk,
|
||||||
prefill_backend=prefill_backend,
|
prefill_backend=prefill_backend,
|
||||||
|
sparse_mla_force_mqa=sparse_mla_force_mqa,
|
||||||
output_scale=output_scale,
|
output_scale=output_scale,
|
||||||
fuse_quant_op=fuse_quant_op,
|
fuse_quant_op=fuse_quant_op,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -205,7 +205,7 @@ hardware and configuration.
|
|||||||
|
|
||||||
| Backend | Description | Dtypes | Compute Cap. | Notes |
|
| Backend | Description | Dtypes | Compute Cap. | Notes |
|
||||||
| ------- | ----------- | ------ | ------------ | ----- |
|
| ------- | ----------- | ------ | ------------ | ----- |
|
||||||
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | FA4 on SM100+, FA3 on SM90, FA2 otherwise |
|
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||||
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
|
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
|
||||||
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||||
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||||
|
|||||||
@@ -304,6 +304,23 @@ def _run_sparse_backend_vs_sdpa(
|
|||||||
scale=kv_cache_scale,
|
scale=kv_cache_scale,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||||
|
# static_forward_context; register a mock layer carrying one.
|
||||||
|
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||||
|
|
||||||
|
prefill_backend = get_mla_prefill_backend(vllm_config)(
|
||||||
|
num_heads=num_heads,
|
||||||
|
scale=scale,
|
||||||
|
kv_lora_rank=kv_lora_rank,
|
||||||
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=qk_rope_head_dim,
|
||||||
|
v_head_dim=v_head_dim,
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
)
|
||||||
|
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||||
|
SimpleNamespace(prefill_backend=prefill_backend)
|
||||||
|
)
|
||||||
|
|
||||||
builder = backend_cls.get_builder_cls()(
|
builder = backend_cls.get_builder_cls()(
|
||||||
kv_cache_spec, ["placeholder"], vllm_config, device
|
kv_cache_spec, ["placeholder"], vllm_config, device
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1031,6 +1031,11 @@ def run_attention_backend(
|
|||||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
||||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||||
@pytest.mark.parametrize("prefill_backend", PREFILL_BACKENDS_TO_TEST)
|
@pytest.mark.parametrize("prefill_backend", PREFILL_BACKENDS_TO_TEST)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("qk_nope_head_dim", "v_head_dim"),
|
||||||
|
[(128, 128), (192, 256)],
|
||||||
|
ids=["deepseek", "glm"],
|
||||||
|
)
|
||||||
def test_backend_correctness(
|
def test_backend_correctness(
|
||||||
default_vllm_config,
|
default_vllm_config,
|
||||||
dist_init,
|
dist_init,
|
||||||
@@ -1042,6 +1047,8 @@ def test_backend_correctness(
|
|||||||
q_scale: float,
|
q_scale: float,
|
||||||
k_scale: float,
|
k_scale: float,
|
||||||
prefill_backend: MLAPrefillBackendEnum,
|
prefill_backend: MLAPrefillBackendEnum,
|
||||||
|
qk_nope_head_dim: int,
|
||||||
|
v_head_dim: int,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Test that all backends produce similar outputs to a reference implementation
|
Test that all backends produce similar outputs to a reference implementation
|
||||||
@@ -1090,9 +1097,9 @@ def test_backend_correctness(
|
|||||||
MLAPrefillSelectorConfig(
|
MLAPrefillSelectorConfig(
|
||||||
dtype=torch.bfloat16,
|
dtype=torch.bfloat16,
|
||||||
mla_dimensions=MLADimensions(
|
mla_dimensions=MLADimensions(
|
||||||
qk_nope_head_dim=128,
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
qk_rope_head_dim=64,
|
qk_rope_head_dim=64,
|
||||||
v_head_dim=128,
|
v_head_dim=v_head_dim,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -1168,8 +1175,6 @@ def test_backend_correctness(
|
|||||||
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
|
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
|
||||||
kv_lora_rank = 512
|
kv_lora_rank = 512
|
||||||
qk_rope_head_dim = 64
|
qk_rope_head_dim = 64
|
||||||
qk_nope_head_dim = 128
|
|
||||||
v_head_dim = 128
|
|
||||||
total_head_size = kv_lora_rank + qk_rope_head_dim
|
total_head_size = kv_lora_rank + qk_rope_head_dim
|
||||||
assert kv_lora_rank + qk_rope_head_dim == head_size, (
|
assert kv_lora_rank + qk_rope_head_dim == head_size, (
|
||||||
f"MLA dimensions don't match: {total_head_size} != {head_size}"
|
f"MLA dimensions don't match: {total_head_size} != {head_size}"
|
||||||
|
|||||||
@@ -442,6 +442,23 @@ def test_sparse_backend_decode_correctness(
|
|||||||
scale=kv_cache_scale,
|
scale=kv_cache_scale,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||||
|
# static_forward_context; register a mock layer carrying one.
|
||||||
|
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||||
|
|
||||||
|
prefill_backend = get_mla_prefill_backend(vllm_config)(
|
||||||
|
num_heads=num_heads,
|
||||||
|
scale=scale,
|
||||||
|
kv_lora_rank=kv_lora_rank,
|
||||||
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=qk_rope_head_dim,
|
||||||
|
v_head_dim=v_head_dim,
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
)
|
||||||
|
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||||
|
SimpleNamespace(prefill_backend=prefill_backend)
|
||||||
|
)
|
||||||
|
|
||||||
builder_cls = backend_cls.get_builder_cls()
|
builder_cls = backend_cls.get_builder_cls()
|
||||||
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||||
metadata = builder.build(
|
metadata = builder.build(
|
||||||
@@ -722,6 +739,258 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
|||||||
assert out == expected
|
assert out == expected
|
||||||
|
|
||||||
|
|
||||||
|
PREFILL_BATCH_SPECS = {
|
||||||
|
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
|
||||||
|
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
torch.cuda.get_device_capability()[0] < 10,
|
||||||
|
reason="Sparse MLA forward_mha requires FA4 (SM100+)",
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
|
||||||
|
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
|
||||||
|
def test_sparse_backend_prefill_correctness(
|
||||||
|
default_vllm_config,
|
||||||
|
dist_init,
|
||||||
|
batch_name,
|
||||||
|
kv_cache_dtype,
|
||||||
|
workspace_init,
|
||||||
|
):
|
||||||
|
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
|
||||||
|
backend_cls = FlashMLASparseBackend
|
||||||
|
batch_spec = PREFILL_BATCH_SPECS[batch_name]
|
||||||
|
|
||||||
|
device = torch.device("cuda")
|
||||||
|
dtype = torch.bfloat16
|
||||||
|
block_size = 64
|
||||||
|
|
||||||
|
num_heads = 128
|
||||||
|
kv_lora_rank = 512
|
||||||
|
qk_nope_head_dim = 128
|
||||||
|
qk_rope_head_dim = 64
|
||||||
|
v_head_dim = 128
|
||||||
|
head_size = kv_lora_rank + qk_rope_head_dim
|
||||||
|
topk_tokens = 512
|
||||||
|
|
||||||
|
max_seqlen = max(batch_spec.seq_lens)
|
||||||
|
total_cache_tokens = sum(batch_spec.seq_lens)
|
||||||
|
|
||||||
|
vllm_config = create_vllm_config(
|
||||||
|
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||||
|
tensor_parallel_size=1,
|
||||||
|
max_model_len=max_seqlen,
|
||||||
|
num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
|
||||||
|
block_size=block_size,
|
||||||
|
hf_config_override={
|
||||||
|
"index_topk": topk_tokens,
|
||||||
|
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
model_config = vllm_config.model_config
|
||||||
|
model_config.hf_text_config = SimpleNamespace(
|
||||||
|
q_lora_rank=None,
|
||||||
|
kv_lora_rank=kv_lora_rank,
|
||||||
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=qk_rope_head_dim,
|
||||||
|
v_head_dim=v_head_dim,
|
||||||
|
model_type="deepseek_v2",
|
||||||
|
)
|
||||||
|
model_config.dtype = dtype
|
||||||
|
model_config.get_num_attention_heads = MethodType(
|
||||||
|
lambda self, parallel_config: num_heads, model_config
|
||||||
|
)
|
||||||
|
model_config.get_num_kv_heads = MethodType(
|
||||||
|
lambda self, parallel_config: 1, model_config
|
||||||
|
)
|
||||||
|
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
|
||||||
|
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
|
||||||
|
|
||||||
|
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
|
||||||
|
scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim)
|
||||||
|
|
||||||
|
torch.manual_seed(42)
|
||||||
|
|
||||||
|
W_UK = torch.rand(
|
||||||
|
kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
|
||||||
|
)
|
||||||
|
W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
seq_lens = batch_spec.seq_lens
|
||||||
|
query_lens = batch_spec.query_lens
|
||||||
|
|
||||||
|
# Compute dense reference outputs.
|
||||||
|
total_query_tokens = sum(query_lens)
|
||||||
|
sparse_indices = torch.zeros(
|
||||||
|
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
all_q, all_kv_c_new, all_k_pe_new = [], [], []
|
||||||
|
kv_c_contexts, k_pe_contexts = [], []
|
||||||
|
reference_outputs = []
|
||||||
|
|
||||||
|
for i in range(batch_spec.batch_size):
|
||||||
|
s_len = seq_lens[i]
|
||||||
|
q_len = query_lens[i]
|
||||||
|
ctx_len = s_len - q_len
|
||||||
|
|
||||||
|
q_mha = torch.rand(
|
||||||
|
q_len,
|
||||||
|
num_heads,
|
||||||
|
qk_nope_head_dim + qk_rope_head_dim,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
|
||||||
|
k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
# Decompress all KV for reference
|
||||||
|
kv_b_weight = torch.cat([W_UK, W_UV], dim=-1).view(
|
||||||
|
kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
|
||||||
|
)
|
||||||
|
kv_decompressed = (kv_c_full @ kv_b_weight).view(
|
||||||
|
s_len, num_heads, qk_nope_head_dim + v_head_dim
|
||||||
|
)
|
||||||
|
k_nope_all, v_all = kv_decompressed.split(
|
||||||
|
[qk_nope_head_dim, v_head_dim], dim=-1
|
||||||
|
)
|
||||||
|
k_pe_expanded = k_pe_full.expand(-1, num_heads, -1)
|
||||||
|
k_all = torch.cat([k_nope_all, k_pe_expanded], dim=-1)
|
||||||
|
|
||||||
|
for j in range(q_len):
|
||||||
|
attend_end = ctx_len + j + 1
|
||||||
|
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
|
||||||
|
k_attend = k_all[:attend_end] # (N, H, D_qk)
|
||||||
|
v_attend = v_all[:attend_end] # (N, H, D_v)
|
||||||
|
|
||||||
|
q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
|
||||||
|
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
|
||||||
|
v_sdpa = v_attend.unsqueeze(0).transpose(1, 2).float()
|
||||||
|
|
||||||
|
out = torch.nn.functional.scaled_dot_product_attention(
|
||||||
|
q_sdpa, k_sdpa, v_sdpa, scale=scale
|
||||||
|
)
|
||||||
|
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
|
||||||
|
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
|
||||||
|
|
||||||
|
all_q.append(q_mha)
|
||||||
|
all_kv_c_new.append(kv_c_full[ctx_len:])
|
||||||
|
all_k_pe_new.append(k_pe_full[ctx_len:])
|
||||||
|
kv_c_contexts.append(kv_c_full)
|
||||||
|
k_pe_contexts.append(k_pe_full)
|
||||||
|
|
||||||
|
query_cat = torch.cat(all_q, dim=0)
|
||||||
|
kv_c_cat = torch.cat(all_kv_c_new, dim=0)
|
||||||
|
k_pe_cat = torch.cat(all_k_pe_new, dim=0)
|
||||||
|
ref_output = torch.cat(reference_outputs, dim=0)
|
||||||
|
|
||||||
|
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||||
|
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||||
|
|
||||||
|
common_attn_metadata = create_common_attn_metadata(
|
||||||
|
batch_spec,
|
||||||
|
vllm_config.cache_config.block_size,
|
||||||
|
device,
|
||||||
|
arange_block_indices=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
kv_cache = create_and_prepopulate_kv_cache(
|
||||||
|
kv_c_contexts=kv_c_contexts,
|
||||||
|
k_pe_contexts=k_pe_contexts,
|
||||||
|
block_size=block_size,
|
||||||
|
head_size=head_size,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
num_blocks=vllm_config.cache_config.num_gpu_blocks,
|
||||||
|
common_attn_metadata=common_attn_metadata,
|
||||||
|
randomize_blocks=False,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||||
|
# static_forward_context; register a mock layer carrying one.
|
||||||
|
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||||
|
|
||||||
|
prefill_backend = get_mla_prefill_backend(vllm_config)(
|
||||||
|
num_heads=num_heads,
|
||||||
|
scale=scale,
|
||||||
|
kv_lora_rank=kv_lora_rank,
|
||||||
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=qk_rope_head_dim,
|
||||||
|
v_head_dim=v_head_dim,
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
)
|
||||||
|
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||||
|
SimpleNamespace(prefill_backend=prefill_backend)
|
||||||
|
)
|
||||||
|
|
||||||
|
builder_cls = backend_cls.get_builder_cls()
|
||||||
|
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||||
|
# Drive the queries through the dense-MHA prefill path directly (the routing
|
||||||
|
# threshold would otherwise classify these short queries as MQA decodes).
|
||||||
|
builder.reorder_batch_threshold = 1
|
||||||
|
metadata = builder.build(
|
||||||
|
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)
|
||||||
|
|
||||||
|
kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1).view(
|
||||||
|
kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_kv_b_proj = ColumnParallelLinear(
|
||||||
|
input_size=kv_lora_rank,
|
||||||
|
output_size=num_heads * (qk_nope_head_dim + v_head_dim),
|
||||||
|
bias=False,
|
||||||
|
).to(device=device, dtype=dtype)
|
||||||
|
mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())
|
||||||
|
|
||||||
|
impl_cls = backend_cls.get_impl_cls()
|
||||||
|
with set_current_vllm_config(vllm_config):
|
||||||
|
impl = impl_cls(
|
||||||
|
num_heads=num_heads,
|
||||||
|
head_size=head_size,
|
||||||
|
scale=scale,
|
||||||
|
num_kv_heads=1,
|
||||||
|
alibi_slopes=None,
|
||||||
|
sliding_window=None,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
logits_soft_cap=None,
|
||||||
|
attn_type="decoder",
|
||||||
|
kv_sharing_target_layer_name=None,
|
||||||
|
q_lora_rank=None,
|
||||||
|
kv_lora_rank=kv_lora_rank,
|
||||||
|
qk_nope_head_dim=qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=qk_rope_head_dim,
|
||||||
|
qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
|
||||||
|
v_head_dim=v_head_dim,
|
||||||
|
kv_b_proj=mock_kv_b_proj,
|
||||||
|
indexer=mock_indexer,
|
||||||
|
)
|
||||||
|
impl.process_weights_after_loading(dtype)
|
||||||
|
|
||||||
|
out_buffer = torch.empty(
|
||||||
|
total_query_tokens, num_heads * v_head_dim, dtype=dtype, device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
with torch.inference_mode():
|
||||||
|
impl.forward_mha(
|
||||||
|
q=query_cat,
|
||||||
|
kv_c_normed=kv_c_cat,
|
||||||
|
k_pe=k_pe_cat,
|
||||||
|
kv_c_and_k_pe_cache=kv_cache,
|
||||||
|
attn_metadata=metadata,
|
||||||
|
k_scale=torch.tensor(1.0, device=device),
|
||||||
|
output=out_buffer,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert out_buffer.shape == ref_output.shape
|
||||||
|
assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
|
||||||
|
torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
|
"seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -59,6 +59,11 @@ class AttentionConfig:
|
|||||||
use_non_causal: bool = False
|
use_non_causal: bool = False
|
||||||
"""Whether to use non-causal (bidirectional) attention."""
|
"""Whether to use non-causal (bidirectional) attention."""
|
||||||
|
|
||||||
|
sparse_mla_force_mqa: bool = False
|
||||||
|
"""Force sparse MLA to use forward_mqa for all requests, including prefill.
|
||||||
|
When False (default), pure prefill batches use forward_mha when implemented.
|
||||||
|
Set to True to always use the MQA path."""
|
||||||
|
|
||||||
flex_attn_block_m: int | None = None
|
flex_attn_block_m: int | None = None
|
||||||
"""Triton kernel BLOCK_M tile size for flex attention.
|
"""Triton kernel BLOCK_M tile size for flex attention.
|
||||||
Must be a power of 2 >= 16. If None and VLLM_BATCH_INVARIANT=1,
|
Must be a power of 2 >= 16. If None and VLLM_BATCH_INVARIANT=1,
|
||||||
|
|||||||
@@ -259,7 +259,6 @@ from vllm.v1.attention.backend import (
|
|||||||
AttentionType,
|
AttentionType,
|
||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
MLAAttentionImpl,
|
MLAAttentionImpl,
|
||||||
SparseMLAAttentionImpl,
|
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.prefill import (
|
from vllm.v1.attention.backends.mla.prefill import (
|
||||||
MLAPrefillBackend,
|
MLAPrefillBackend,
|
||||||
@@ -490,16 +489,30 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
|||||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||||
compilation_config.static_forward_context[prefix] = self
|
compilation_config.static_forward_context[prefix] = self
|
||||||
|
|
||||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
self.prefill_backend: MLAPrefillBackend | None
|
||||||
self.prefill_backend = prefill_backend_cls(
|
try:
|
||||||
num_heads=self.num_heads,
|
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||||
scale=self.scale,
|
except ValueError:
|
||||||
kv_lora_rank=self.kv_lora_rank,
|
if (
|
||||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
not self.impl.is_sparse
|
||||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
or vllm_config.attention_config.mla_prefill_backend is not None
|
||||||
v_head_dim=self.v_head_dim,
|
):
|
||||||
vllm_config=vllm_config,
|
raise
|
||||||
)
|
logger.warning_once(
|
||||||
|
"No MLA prefill backend supports this model; sparse MLA will use the "
|
||||||
|
"top-k MQA path only (no dense-MHA prefill)."
|
||||||
|
)
|
||||||
|
self.prefill_backend = None
|
||||||
|
else:
|
||||||
|
self.prefill_backend = prefill_backend_cls(
|
||||||
|
num_heads=self.num_heads,
|
||||||
|
scale=self.scale,
|
||||||
|
kv_lora_rank=self.kv_lora_rank,
|
||||||
|
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||||
|
v_head_dim=self.v_head_dim,
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
)
|
||||||
|
|
||||||
self.kv_cache = torch.tensor([])
|
self.kv_cache = torch.tensor([])
|
||||||
|
|
||||||
@@ -693,23 +706,28 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
|||||||
if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla":
|
if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla":
|
||||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||||
|
|
||||||
# Sparse MLA impls only support forward_mqa (decode-style attention)
|
assert (
|
||||||
is_sparse_impl = isinstance(self.impl, SparseMLAAttentionImpl)
|
attn_metadata.num_decodes is not None
|
||||||
|
and attn_metadata.num_prefills is not None
|
||||||
|
and attn_metadata.num_decode_tokens is not None
|
||||||
|
)
|
||||||
|
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||||
|
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||||
|
|
||||||
if is_sparse_impl:
|
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||||
num_mqa_tokens = q.size(0)
|
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||||
num_mha_tokens = 0
|
use_mha = (
|
||||||
else:
|
self.prefill_backend is not None
|
||||||
assert (
|
and prefill_max_seq_len <= attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||||
attn_metadata.num_decodes is not None
|
and not self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||||
and attn_metadata.num_prefills is not None
|
|
||||||
and attn_metadata.num_decode_tokens is not None
|
|
||||||
)
|
)
|
||||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
if not use_mha:
|
||||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
num_mqa_tokens = q.size(0)
|
||||||
|
num_mha_tokens = 0
|
||||||
|
|
||||||
mha_use_quant_output = (
|
mha_use_quant_output = (
|
||||||
quant_key is not None
|
quant_key is not None
|
||||||
|
and self.prefill_backend is not None
|
||||||
and self.prefill_backend.supports_quant_output(quant_key)
|
and self.prefill_backend.supports_quant_output(quant_key)
|
||||||
and attn_metadata is not None
|
and attn_metadata is not None
|
||||||
and attn_metadata.prefill is not None
|
and attn_metadata.prefill is not None
|
||||||
@@ -807,7 +825,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
|||||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||||
|
|
||||||
# call decode attn
|
# call decode attn
|
||||||
if not is_sparse_impl:
|
if not self.impl.is_sparse:
|
||||||
assert attn_metadata.decode is not None
|
assert attn_metadata.decode is not None
|
||||||
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined]
|
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined]
|
||||||
|
|
||||||
@@ -1398,6 +1416,182 @@ def backend_supports_prefill_query_quantization() -> bool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_mla_chunked_context_metadata(
|
||||||
|
*,
|
||||||
|
context_lens_cpu: torch.Tensor,
|
||||||
|
prefill_query_start_loc_cpu: torch.Tensor,
|
||||||
|
num_prefills: int,
|
||||||
|
chunked_prefill_workspace: torch.Tensor,
|
||||||
|
chunked_prefill_workspace_size: int,
|
||||||
|
block_size: int,
|
||||||
|
align_chunk_to_block: bool,
|
||||||
|
device: torch.device,
|
||||||
|
dcp_world_size: int,
|
||||||
|
dcp_local_block_size: int,
|
||||||
|
dcp_virtual_block_size: int,
|
||||||
|
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||||
|
"""Build chunked-context metadata for an MLA prefill.
|
||||||
|
|
||||||
|
Shared by dense and sparse builders. Splits each prefill's context
|
||||||
|
into workspace-sized chunks and, under DCP, plans the per-rank interleaved
|
||||||
|
local chunks the all-gather reduction consumes.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
context_lens_cpu: Per-prefill context length (seq_len - query_len).
|
||||||
|
prefill_query_start_loc_cpu: Prefill query cumulative offsets (0-based).
|
||||||
|
num_prefills: Number of prefill requests.
|
||||||
|
chunked_prefill_workspace: Scratch buffer the context gather writes to.
|
||||||
|
chunked_prefill_workspace_size: Row capacity of the workspace.
|
||||||
|
block_size: KV cache page size for chunk-start alignment.
|
||||||
|
align_chunk_to_block: Round the chunk size down to ``block_size``.
|
||||||
|
device: Target device for the returned tensors.
|
||||||
|
dcp_world_size: Decode-context-parallel world size (1 if disabled).
|
||||||
|
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||||
|
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The chunked-context metadata, or None when no prefill has any context.
|
||||||
|
"""
|
||||||
|
# NOTE: it is recommended you read the `Chunked Prefill` section in the
|
||||||
|
# comment at the top of the file before trying to understand this code.
|
||||||
|
max_context_len = context_lens_cpu.max().item()
|
||||||
|
if max_context_len <= 0:
|
||||||
|
return None
|
||||||
|
num_prefills_with_context = int((context_lens_cpu > 0).sum().item())
|
||||||
|
|
||||||
|
# Currently we allocate an equal amount of workspace for each prefill with
|
||||||
|
# context; we could probably use a more advanced algorithm here and allocate
|
||||||
|
# more workspace to prefills with longer context lengths.
|
||||||
|
max_context_chunk = chunked_prefill_workspace_size // num_prefills_with_context
|
||||||
|
if align_chunk_to_block:
|
||||||
|
# The `gather_and_maybe_dequant_cache` kernel cannot handle chunk
|
||||||
|
# starts that are not aligned to block_size, so round down.
|
||||||
|
max_context_chunk = round_down(max_context_chunk, block_size)
|
||||||
|
assert max_context_chunk > 0
|
||||||
|
|
||||||
|
num_chunks = cdiv(max_context_len, max_context_chunk)
|
||||||
|
# e.g. max_context_chunk=256, num_chunks=3, num_prefills=4 ->
|
||||||
|
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
||||||
|
# Note(simon): this is done on CPU because of downstream's use of `to_list`.
|
||||||
|
chunk_starts = torch.empty(
|
||||||
|
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||||
|
).copy_(
|
||||||
|
torch.arange(num_chunks, dtype=torch.int32)
|
||||||
|
.multiply_(max_context_chunk)
|
||||||
|
.unsqueeze(1)
|
||||||
|
)
|
||||||
|
chunk_ends = torch.min(
|
||||||
|
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
||||||
|
)
|
||||||
|
chunk_seq_lens = (chunk_ends - chunk_starts).clamp(min=0)
|
||||||
|
|
||||||
|
cu_seq_lens_cpu = torch.zeros(
|
||||||
|
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||||
|
)
|
||||||
|
torch.cumsum(chunk_seq_lens, dim=1, out=cu_seq_lens_cpu[:, 1:], dtype=torch.int32)
|
||||||
|
chunk_total_token = cu_seq_lens_cpu[:, -1]
|
||||||
|
|
||||||
|
max_tokens_over_chunk = chunk_total_token.max().item()
|
||||||
|
token_to_seq_cpu = torch.zeros(
|
||||||
|
(num_chunks, max_tokens_over_chunk), dtype=torch.int32, pin_memory=True
|
||||||
|
)
|
||||||
|
req_indices = torch.arange(num_prefills, dtype=torch.int32)
|
||||||
|
for i in range(num_chunks):
|
||||||
|
token_to_seq = torch.repeat_interleave(req_indices, chunk_seq_lens[i])
|
||||||
|
token_to_seq_cpu[i, : token_to_seq.shape[0]] = token_to_seq
|
||||||
|
|
||||||
|
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
||||||
|
num_prefills_with_context
|
||||||
|
].item()
|
||||||
|
|
||||||
|
metadata_cls = MLACommonPrefillMetadata.ChunkedContextMetadata
|
||||||
|
if dcp_world_size > 1:
|
||||||
|
local_context_lens_allranks = get_dcp_local_seq_lens(
|
||||||
|
context_lens_cpu, dcp_world_size, None, dcp_local_block_size
|
||||||
|
)
|
||||||
|
# Note(qcs): Per-rank local context lengths, padded to
|
||||||
|
# `dcp_local_block_size`.
|
||||||
|
padded_local_context_lens_cpu: torch.Tensor = (
|
||||||
|
cdiv(context_lens_cpu, dcp_virtual_block_size) * dcp_local_block_size
|
||||||
|
)
|
||||||
|
# Note(hc): The above max_context_chunk already enforces block_size
|
||||||
|
# alignment; DCP only requires block_size be divisible by dcp_world_size,
|
||||||
|
# because DCP uses cp_gather_cache, which does not require chunk starts
|
||||||
|
# aligned to block_size.
|
||||||
|
assert max_context_chunk % dcp_world_size == 0
|
||||||
|
padded_local_max_context_chunk = (
|
||||||
|
cdiv(max_context_chunk, dcp_virtual_block_size) * dcp_local_block_size
|
||||||
|
)
|
||||||
|
local_chunk_starts = torch.empty(
|
||||||
|
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||||
|
).copy_(
|
||||||
|
torch.arange(num_chunks, dtype=torch.int32)
|
||||||
|
.multiply_(padded_local_max_context_chunk)
|
||||||
|
.unsqueeze(1)
|
||||||
|
)
|
||||||
|
local_chunk_ends = torch.min(
|
||||||
|
padded_local_context_lens_cpu.unsqueeze(0),
|
||||||
|
local_chunk_starts + padded_local_max_context_chunk,
|
||||||
|
)
|
||||||
|
padded_local_chunk_seq_lens = (local_chunk_ends - local_chunk_starts).clamp(
|
||||||
|
min=0
|
||||||
|
)
|
||||||
|
|
||||||
|
padded_local_cu_seq_lens_cpu = torch.zeros(
|
||||||
|
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||||
|
)
|
||||||
|
torch.cumsum(
|
||||||
|
padded_local_chunk_seq_lens,
|
||||||
|
dim=1,
|
||||||
|
out=padded_local_cu_seq_lens_cpu[:, 1:],
|
||||||
|
dtype=torch.int32,
|
||||||
|
)
|
||||||
|
max_padded_local_tokens = padded_local_cu_seq_lens_cpu[:, -1].max().item()
|
||||||
|
padded_local_token_to_seq_cpu = torch.zeros(
|
||||||
|
(num_chunks, max_padded_local_tokens), dtype=torch.int32
|
||||||
|
)
|
||||||
|
for i in range(num_chunks):
|
||||||
|
tts = torch.repeat_interleave(req_indices, padded_local_chunk_seq_lens[i])
|
||||||
|
padded_local_token_to_seq_cpu[i, : tts.shape[0]] = tts
|
||||||
|
|
||||||
|
chunked_context_metadata = metadata_cls(
|
||||||
|
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||||
|
starts=local_chunk_starts.to(device, non_blocking=True),
|
||||||
|
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
||||||
|
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||||
|
seq_lens=chunk_seq_lens,
|
||||||
|
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||||
|
chunk_total_token=chunk_total_token.tolist(),
|
||||||
|
workspace=chunked_prefill_workspace,
|
||||||
|
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||||
|
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
|
||||||
|
local_context_lens_allranks=local_context_lens_allranks.tolist(),
|
||||||
|
padded_local_cu_seq_lens=padded_local_cu_seq_lens_cpu.to(
|
||||||
|
device, non_blocking=True
|
||||||
|
),
|
||||||
|
padded_local_token_to_seq=padded_local_token_to_seq_cpu.to(
|
||||||
|
device, non_blocking=True
|
||||||
|
),
|
||||||
|
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||||
|
chunk_size=padded_local_max_context_chunk,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
chunked_context_metadata = metadata_cls(
|
||||||
|
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||||
|
starts=chunk_starts.to(device, non_blocking=True),
|
||||||
|
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
||||||
|
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||||
|
seq_lens=chunk_seq_lens,
|
||||||
|
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||||
|
chunk_total_token=chunk_total_token,
|
||||||
|
workspace=chunked_prefill_workspace,
|
||||||
|
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert max(chunked_context_metadata.max_seq_lens) <= chunked_prefill_workspace_size
|
||||||
|
return chunked_context_metadata
|
||||||
|
|
||||||
|
|
||||||
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||||
"""
|
"""
|
||||||
NOTE: Please read the comment at the top of the file before trying to
|
NOTE: Please read the comment at the top of the file before trying to
|
||||||
@@ -1662,8 +1856,6 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
|||||||
context_lens_cpu = (
|
context_lens_cpu = (
|
||||||
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
||||||
)
|
)
|
||||||
max_context_len_cpu = context_lens_cpu.max().item()
|
|
||||||
num_prefills_with_context_cpu = (context_lens_cpu > 0).sum().item()
|
|
||||||
prefill_query_start_loc = (
|
prefill_query_start_loc = (
|
||||||
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
||||||
)
|
)
|
||||||
@@ -1671,188 +1863,19 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
|||||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||||
)
|
)
|
||||||
|
|
||||||
chunked_context_metadata = None
|
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||||
if max_context_len_cpu > 0:
|
context_lens_cpu=context_lens_cpu,
|
||||||
# NOTE: it is recommend you read the `Chunked Prefill` section
|
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||||
# in the comment at the top of the file before trying to
|
num_prefills=num_prefills,
|
||||||
# understand the following code
|
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||||
|
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||||
# currently we allocate an equal amount of workspace for each
|
block_size=self.page_size,
|
||||||
# prefill in the batch, we could probably use a more advanced
|
align_chunk_to_block=True,
|
||||||
# algorithm here and allocate more workspace to prefills with
|
device=device,
|
||||||
# longer context lengths
|
dcp_world_size=self.dcp_world_size,
|
||||||
max_context_chunk = (
|
dcp_local_block_size=self.dcp_local_block_size,
|
||||||
self.chunked_prefill_workspace_size // num_prefills_with_context_cpu
|
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||||
)
|
)
|
||||||
|
|
||||||
# align max_context_chunk to page_size by rounding down,
|
|
||||||
# currently the `gather_and_maybe_dequant_cache` kernel
|
|
||||||
# cannot handle `context_chunk_starts` that are not aligned
|
|
||||||
# to page_size
|
|
||||||
max_context_chunk = round_down(max_context_chunk, self.page_size)
|
|
||||||
|
|
||||||
assert max_context_chunk > 0
|
|
||||||
num_chunks = cdiv(max_context_len_cpu, max_context_chunk)
|
|
||||||
|
|
||||||
# if `max_context_chunk = 256`, `num_chunks = 3`, and
|
|
||||||
# `num_prefills_with_context = 4`, create a tensor that looks
|
|
||||||
# like
|
|
||||||
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
|
||||||
# Note(simon): this is done in CPU because of downstream's
|
|
||||||
# of `to_list`.
|
|
||||||
chunk_starts = torch.empty(
|
|
||||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
|
||||||
).copy_(
|
|
||||||
torch.arange(num_chunks, dtype=torch.int32)
|
|
||||||
.multiply_(max_context_chunk)
|
|
||||||
.unsqueeze(1)
|
|
||||||
)
|
|
||||||
chunk_ends = torch.min(
|
|
||||||
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
|
||||||
)
|
|
||||||
chunk_seq_lens = (chunk_ends - chunk_starts).clamp(min=0)
|
|
||||||
|
|
||||||
cu_seq_lens_cpu = torch.zeros(
|
|
||||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
|
||||||
)
|
|
||||||
torch.cumsum(
|
|
||||||
chunk_seq_lens, dim=1, out=cu_seq_lens_cpu[:, 1:], dtype=torch.int32
|
|
||||||
)
|
|
||||||
chunk_total_token = cu_seq_lens_cpu[:, -1]
|
|
||||||
|
|
||||||
max_token_num_over_chunk = chunk_total_token.max().item()
|
|
||||||
token_to_seq_tensor_cpu = torch.zeros(
|
|
||||||
[num_chunks, max_token_num_over_chunk],
|
|
||||||
dtype=torch.int32,
|
|
||||||
pin_memory=True,
|
|
||||||
)
|
|
||||||
range_idx = torch.arange(num_prefills, dtype=torch.int32)
|
|
||||||
for i in range(num_chunks):
|
|
||||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
|
||||||
range_idx, chunk_seq_lens[i]
|
|
||||||
)
|
|
||||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
|
||||||
token_to_seq_tensor_cpu[i, :chunk_len] = chunk_token_to_seq_tensor
|
|
||||||
|
|
||||||
if self.dcp_world_size > 1:
|
|
||||||
local_context_lens_allranks = get_dcp_local_seq_lens(
|
|
||||||
context_lens_cpu,
|
|
||||||
self.dcp_world_size,
|
|
||||||
None,
|
|
||||||
self.dcp_local_block_size,
|
|
||||||
)
|
|
||||||
# Note(qcs): The max local context lengths
|
|
||||||
# padded to `dcp_local_block_size`.
|
|
||||||
padded_local_context_lens_cpu: torch.Tensor = (
|
|
||||||
cdiv(
|
|
||||||
context_lens_cpu,
|
|
||||||
self.dcp_virtual_block_size,
|
|
||||||
)
|
|
||||||
* self.dcp_local_block_size
|
|
||||||
)
|
|
||||||
# Note(hc): The above max_context_chunk already enforces
|
|
||||||
# block_size alignment, DCP just need the block_size can
|
|
||||||
# be divisible by dcp_world_size, because DCP use
|
|
||||||
# cp_gather_cache which not require `cp_chunk_starts`
|
|
||||||
# aligned to page_size.
|
|
||||||
assert max_context_chunk % self.dcp_world_size == 0
|
|
||||||
padded_local_max_context_chunk_across_ranks = (
|
|
||||||
cdiv(
|
|
||||||
max_context_chunk,
|
|
||||||
self.dcp_virtual_block_size,
|
|
||||||
)
|
|
||||||
* self.dcp_local_block_size
|
|
||||||
)
|
|
||||||
local_chunk_starts = torch.empty(
|
|
||||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
|
||||||
).copy_(
|
|
||||||
torch.arange(num_chunks, dtype=torch.int32)
|
|
||||||
.multiply_(padded_local_max_context_chunk_across_ranks)
|
|
||||||
.unsqueeze(1)
|
|
||||||
)
|
|
||||||
local_chunk_ends = torch.min(
|
|
||||||
padded_local_context_lens_cpu.unsqueeze(0),
|
|
||||||
local_chunk_starts
|
|
||||||
+ padded_local_max_context_chunk_across_ranks,
|
|
||||||
)
|
|
||||||
padded_local_chunk_seq_lens = (
|
|
||||||
local_chunk_ends - local_chunk_starts
|
|
||||||
).clamp(min=0)
|
|
||||||
|
|
||||||
padded_local_cu_chunk_seq_lens_cpu = torch.zeros(
|
|
||||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
|
||||||
)
|
|
||||||
torch.cumsum(
|
|
||||||
padded_local_chunk_seq_lens,
|
|
||||||
dim=1,
|
|
||||||
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
|
|
||||||
dtype=torch.int32,
|
|
||||||
)
|
|
||||||
max_padded_local_tokens_over_chunk = (
|
|
||||||
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
|
|
||||||
)
|
|
||||||
padded_local_token_to_seq_tensor_cpu = torch.zeros(
|
|
||||||
[num_chunks, max_padded_local_tokens_over_chunk],
|
|
||||||
dtype=torch.int32,
|
|
||||||
)
|
|
||||||
for i in range(num_chunks):
|
|
||||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
|
||||||
range_idx, padded_local_chunk_seq_lens[i]
|
|
||||||
)
|
|
||||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
|
||||||
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
|
|
||||||
chunk_token_to_seq_tensor
|
|
||||||
)
|
|
||||||
|
|
||||||
prefill_tokens_with_context = None
|
|
||||||
if num_prefills_with_context_cpu > 0:
|
|
||||||
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
|
||||||
num_prefills_with_context_cpu
|
|
||||||
].item()
|
|
||||||
_ChunkedMetadata = MLACommonPrefillMetadata.ChunkedContextMetadata
|
|
||||||
if self.dcp_world_size > 1:
|
|
||||||
chunked_context_metadata = _ChunkedMetadata(
|
|
||||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
|
||||||
starts=local_chunk_starts.to(device, non_blocking=True),
|
|
||||||
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
|
||||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
|
||||||
seq_lens=chunk_seq_lens,
|
|
||||||
token_to_seq=token_to_seq_tensor_cpu.to(
|
|
||||||
device, non_blocking=True
|
|
||||||
),
|
|
||||||
chunk_total_token=chunk_total_token.tolist(),
|
|
||||||
workspace=self.chunked_prefill_workspace,
|
|
||||||
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
|
|
||||||
local_context_lens_allranks=local_context_lens_allranks.tolist(),
|
|
||||||
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
|
|
||||||
device, non_blocking=True
|
|
||||||
),
|
|
||||||
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
|
|
||||||
device, non_blocking=True
|
|
||||||
),
|
|
||||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
|
||||||
chunk_size=padded_local_max_context_chunk_across_ranks,
|
|
||||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
chunked_context_metadata = _ChunkedMetadata(
|
|
||||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
|
||||||
starts=chunk_starts.to(device, non_blocking=True),
|
|
||||||
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
|
||||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
|
||||||
seq_lens=chunk_seq_lens,
|
|
||||||
token_to_seq=token_to_seq_tensor_cpu.to(
|
|
||||||
device, non_blocking=True
|
|
||||||
),
|
|
||||||
chunk_total_token=chunk_total_token,
|
|
||||||
workspace=self.chunked_prefill_workspace,
|
|
||||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (
|
|
||||||
max(chunked_context_metadata.max_seq_lens)
|
|
||||||
<= self.chunked_prefill_workspace_size
|
|
||||||
)
|
|
||||||
|
|
||||||
prefill_metadata = MLACommonPrefillMetadata(
|
prefill_metadata = MLACommonPrefillMetadata(
|
||||||
block_table=block_table_tensor[reqs_start:, ...],
|
block_table=block_table_tensor[reqs_start:, ...],
|
||||||
@@ -1985,19 +2008,14 @@ def reorg_kvcache(
|
|||||||
return reorganized_kv_c_normed, reorganized_k_pe
|
return reorganized_kv_c_normed, reorganized_k_pe
|
||||||
|
|
||||||
|
|
||||||
class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||||
"""
|
"""
|
||||||
NOTE: Please read the comment at the top of the file before trying to
|
Shared MLA base providing dense-MHA prefill (via the selected
|
||||||
understand this class
|
MLAPrefillBackend) for both dense and sparse impls; subclasses add decode
|
||||||
|
(``forward_mqa``).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def fused_output_quant_supported(self, quant_key):
|
_use_flashinfer_concat_mla_k: bool
|
||||||
return quant_key in (
|
|
||||||
kFp8StaticTensorSym,
|
|
||||||
kNvfp4Dynamic,
|
|
||||||
kFp8Dynamic128Sym,
|
|
||||||
kFp8Dynamic64Sym,
|
|
||||||
)
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -2005,60 +2023,25 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
|||||||
head_size: int,
|
head_size: int,
|
||||||
scale: float,
|
scale: float,
|
||||||
num_kv_heads: int,
|
num_kv_heads: int,
|
||||||
alibi_slopes: list[float] | None,
|
|
||||||
sliding_window: int | None,
|
|
||||||
kv_cache_dtype: str,
|
kv_cache_dtype: str,
|
||||||
logits_soft_cap: float | None,
|
|
||||||
attn_type: str,
|
|
||||||
kv_sharing_target_layer_name: str | None,
|
|
||||||
# MLA Specific Arguments
|
|
||||||
q_lora_rank: int | None,
|
|
||||||
kv_lora_rank: int,
|
kv_lora_rank: int,
|
||||||
qk_nope_head_dim: int,
|
qk_nope_head_dim: int,
|
||||||
qk_rope_head_dim: int,
|
qk_rope_head_dim: int,
|
||||||
qk_head_dim: int,
|
qk_head_dim: int,
|
||||||
v_head_dim: int,
|
v_head_dim: int,
|
||||||
kv_b_proj: ColumnParallelLinear,
|
kv_b_proj: ColumnParallelLinear,
|
||||||
# DSV3.2 MLA Specific Arguments
|
|
||||||
indexer: object | None = None,
|
|
||||||
q_pad_num_heads: int | None = None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
if kv_sharing_target_layer_name is not None:
|
|
||||||
raise NotImplementedError("KV sharing is not supported for MLA")
|
|
||||||
|
|
||||||
self.num_heads = num_heads
|
self.num_heads = num_heads
|
||||||
self.head_size = head_size
|
self.head_size = head_size
|
||||||
self.scale = float(scale)
|
self.scale = float(scale)
|
||||||
self.num_kv_heads = num_kv_heads
|
self.num_kv_heads = num_kv_heads
|
||||||
self.kv_cache_dtype = kv_cache_dtype
|
self.kv_cache_dtype = kv_cache_dtype
|
||||||
|
|
||||||
self.q_lora_rank = q_lora_rank
|
|
||||||
self.kv_lora_rank = kv_lora_rank
|
self.kv_lora_rank = kv_lora_rank
|
||||||
self.qk_nope_head_dim = qk_nope_head_dim
|
self.qk_nope_head_dim = qk_nope_head_dim
|
||||||
self.qk_rope_head_dim = qk_rope_head_dim
|
self.qk_rope_head_dim = qk_rope_head_dim
|
||||||
self.qk_head_dim = qk_head_dim
|
self.qk_head_dim = qk_head_dim
|
||||||
self.v_head_dim = v_head_dim
|
self.v_head_dim = v_head_dim
|
||||||
self.kv_b_proj = kv_b_proj
|
self.kv_b_proj = kv_b_proj
|
||||||
self.indexer = indexer
|
|
||||||
self.q_pad_num_heads = q_pad_num_heads
|
|
||||||
self.supports_quant_query_input = True
|
|
||||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
|
||||||
|
|
||||||
# Use flashinfer's optimized concat_mla_k kernel when available.
|
|
||||||
# The kernel is optimized for DeepSeek V3 dimensions:
|
|
||||||
# num_heads=128, nope_dim=128, rope_dim=64
|
|
||||||
self._use_flashinfer_concat_mla_k = (
|
|
||||||
has_flashinfer()
|
|
||||||
and (self.num_heads == 128)
|
|
||||||
and (self.qk_nope_head_dim == 128)
|
|
||||||
and (self.qk_rope_head_dim == 64)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.dcp_world_size: int = -1
|
|
||||||
|
|
||||||
self.cp_kv_cache_interleave_size: int = (
|
|
||||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
|
||||||
)
|
|
||||||
|
|
||||||
def _concat_k_nope_k_pe(
|
def _concat_k_nope_k_pe(
|
||||||
self, k_nope: torch.Tensor, k_pe: torch.Tensor
|
self, k_nope: torch.Tensor, k_pe: torch.Tensor
|
||||||
@@ -2341,7 +2324,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
|||||||
|
|
||||||
return output, output_lse
|
return output, output_lse
|
||||||
|
|
||||||
def forward_mha(
|
def forward_mha( # type: ignore[override]
|
||||||
self,
|
self,
|
||||||
q: torch.Tensor,
|
q: torch.Tensor,
|
||||||
kv_c_normed: torch.Tensor,
|
kv_c_normed: torch.Tensor,
|
||||||
@@ -2428,6 +2411,83 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
|||||||
output_prefill = output_prefill.flatten(start_dim=-2)
|
output_prefill = output_prefill.flatten(start_dim=-2)
|
||||||
output.copy_(output_prefill)
|
output.copy_(output_prefill)
|
||||||
|
|
||||||
|
|
||||||
|
class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||||
|
"""
|
||||||
|
NOTE: Please read the comment at the top of the file before trying to
|
||||||
|
understand this class
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fused_output_quant_supported(self, quant_key):
|
||||||
|
return quant_key in (
|
||||||
|
kFp8StaticTensorSym,
|
||||||
|
kNvfp4Dynamic,
|
||||||
|
kFp8Dynamic128Sym,
|
||||||
|
kFp8Dynamic64Sym,
|
||||||
|
)
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
num_heads: int,
|
||||||
|
head_size: int,
|
||||||
|
scale: float,
|
||||||
|
num_kv_heads: int,
|
||||||
|
alibi_slopes: list[float] | None,
|
||||||
|
sliding_window: int | None,
|
||||||
|
kv_cache_dtype: str,
|
||||||
|
logits_soft_cap: float | None,
|
||||||
|
attn_type: str,
|
||||||
|
kv_sharing_target_layer_name: str | None,
|
||||||
|
# MLA Specific Arguments
|
||||||
|
q_lora_rank: int | None,
|
||||||
|
kv_lora_rank: int,
|
||||||
|
qk_nope_head_dim: int,
|
||||||
|
qk_rope_head_dim: int,
|
||||||
|
qk_head_dim: int,
|
||||||
|
v_head_dim: int,
|
||||||
|
kv_b_proj: ColumnParallelLinear,
|
||||||
|
# DSV3.2 MLA Specific Arguments
|
||||||
|
indexer: object | None = None,
|
||||||
|
q_pad_num_heads: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
if kv_sharing_target_layer_name is not None:
|
||||||
|
raise NotImplementedError("KV sharing is not supported for MLA")
|
||||||
|
|
||||||
|
super().__init__(
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
scale,
|
||||||
|
num_kv_heads,
|
||||||
|
kv_cache_dtype,
|
||||||
|
kv_lora_rank,
|
||||||
|
qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim,
|
||||||
|
qk_head_dim,
|
||||||
|
v_head_dim,
|
||||||
|
kv_b_proj,
|
||||||
|
)
|
||||||
|
self.q_lora_rank = q_lora_rank
|
||||||
|
self.indexer = indexer
|
||||||
|
self.q_pad_num_heads = q_pad_num_heads
|
||||||
|
self.supports_quant_query_input = True
|
||||||
|
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||||
|
|
||||||
|
# Use flashinfer's optimized concat_mla_k kernel when available.
|
||||||
|
# The kernel is optimized for DeepSeek V3 dimensions:
|
||||||
|
# num_heads=128, nope_dim=128, rope_dim=64
|
||||||
|
self._use_flashinfer_concat_mla_k = (
|
||||||
|
has_flashinfer()
|
||||||
|
and (self.num_heads == 128)
|
||||||
|
and (self.qk_nope_head_dim == 128)
|
||||||
|
and (self.qk_rope_head_dim == 64)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.dcp_world_size: int = -1
|
||||||
|
|
||||||
|
self.cp_kv_cache_interleave_size: int = (
|
||||||
|
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||||
|
)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def forward_mqa(
|
def forward_mqa(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -0,0 +1,316 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
|
||||||
|
|
||||||
|
from shutil import which
|
||||||
|
from typing import TYPE_CHECKING, Generic, TypeVar
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.distributed import get_dcp_group
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.model_executor.layers.attention.mla_attention import (
|
||||||
|
MLACommonBaseImpl,
|
||||||
|
MLACommonPrefillMetadata,
|
||||||
|
build_mla_chunked_context_metadata,
|
||||||
|
get_mla_dims,
|
||||||
|
)
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.flashinfer import has_flashinfer
|
||||||
|
from vllm.utils.torch_utils import np_to_pinned_tensor
|
||||||
|
from vllm.v1.attention.backend import (
|
||||||
|
AttentionMetadata,
|
||||||
|
AttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||||
|
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||||
|
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
T = TypeVar("T", bound=AttentionMetadata)
|
||||||
|
|
||||||
|
|
||||||
|
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||||
|
metadata_cls: type[T]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
kv_cache_spec: "AttentionSpec",
|
||||||
|
layer_names: list[str],
|
||||||
|
vllm_config: "VllmConfig",
|
||||||
|
device: torch.device,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
|
self.vllm_config = vllm_config
|
||||||
|
self.device = device
|
||||||
|
self.model_config = vllm_config.model_config
|
||||||
|
self.mla_dims = get_mla_dims(self.model_config)
|
||||||
|
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
|
||||||
|
self.req_id_per_token_buffer = torch.empty(
|
||||||
|
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
parallel_config = vllm_config.parallel_config
|
||||||
|
try:
|
||||||
|
self.dcp_world_size = get_dcp_group().world_size
|
||||||
|
self.dcp_rank = get_dcp_group().rank_in_group
|
||||||
|
except AssertionError:
|
||||||
|
# DCP might not be initialized in testing
|
||||||
|
self.dcp_world_size = 1
|
||||||
|
self.dcp_rank = 0
|
||||||
|
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||||
|
self.dcp_local_block_size = self.cp_kv_cache_interleave_size
|
||||||
|
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
||||||
|
|
||||||
|
self.chunked_prefill_workspace_size = (
|
||||||
|
self.determine_chunked_prefill_workspace_size(vllm_config)
|
||||||
|
)
|
||||||
|
workspace_head_size = (
|
||||||
|
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim
|
||||||
|
)
|
||||||
|
workspace_rows = self.chunked_prefill_workspace_size
|
||||||
|
if self.dcp_world_size > 1:
|
||||||
|
# DCP gathers each rank's local KV shard into the workspace, so it
|
||||||
|
# needs an extra 1/DCP rows beyond the TP allocation.
|
||||||
|
assert self.chunked_prefill_workspace_size % self.dcp_world_size == 0
|
||||||
|
workspace_rows += self.chunked_prefill_workspace_size // self.dcp_world_size
|
||||||
|
self.chunked_prefill_workspace = torch.empty(
|
||||||
|
(workspace_rows, workspace_head_size),
|
||||||
|
dtype=self.model_config.dtype,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
|
||||||
|
layer_names[0]
|
||||||
|
].prefill_backend
|
||||||
|
self._prefill_backend = (
|
||||||
|
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def determine_chunked_prefill_workspace_size(vllm_config: "VllmConfig") -> int:
|
||||||
|
scheduler_config = vllm_config.scheduler_config
|
||||||
|
cache_config = vllm_config.cache_config
|
||||||
|
model_config = vllm_config.model_config
|
||||||
|
topk_tokens = model_config.hf_config.index_topk
|
||||||
|
|
||||||
|
workspace_size = min(
|
||||||
|
max(
|
||||||
|
8 * model_config.max_model_len,
|
||||||
|
4 * scheduler_config.max_num_seqs * cache_config.block_size,
|
||||||
|
),
|
||||||
|
64 * 1024,
|
||||||
|
scheduler_config.max_num_seqs * topk_tokens,
|
||||||
|
)
|
||||||
|
return max(
|
||||||
|
workspace_size,
|
||||||
|
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _build_req_id_per_token(
|
||||||
|
self,
|
||||||
|
common_attn_metadata: "CommonAttentionMetadata",
|
||||||
|
) -> torch.Tensor:
|
||||||
|
num_tokens = common_attn_metadata.num_actual_tokens
|
||||||
|
starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
|
||||||
|
seg_lengths = np.diff(starts)
|
||||||
|
req_id_per_token = np.repeat(
|
||||||
|
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
||||||
|
)
|
||||||
|
self.req_id_per_token_buffer.fill_(0)
|
||||||
|
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
||||||
|
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||||
|
)
|
||||||
|
return self.req_id_per_token_buffer[:num_tokens]
|
||||||
|
|
||||||
|
def _build_chunked_context_fields(
|
||||||
|
self,
|
||||||
|
common_attn_metadata: "CommonAttentionMetadata",
|
||||||
|
num_decodes: int,
|
||||||
|
num_prefills: int,
|
||||||
|
prefill_query_lens_cpu: torch.Tensor | None,
|
||||||
|
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||||
|
if num_prefills == 0 or prefill_query_lens_cpu is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||||
|
assert seq_lens_cpu is not None
|
||||||
|
context_lens_cpu = (
|
||||||
|
seq_lens_cpu[num_decodes : num_decodes + num_prefills]
|
||||||
|
- prefill_query_lens_cpu
|
||||||
|
)
|
||||||
|
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||||
|
prefill_query_start_loc_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]
|
||||||
|
|
||||||
|
return build_mla_chunked_context_metadata(
|
||||||
|
context_lens_cpu=context_lens_cpu,
|
||||||
|
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||||
|
num_prefills=num_prefills,
|
||||||
|
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||||
|
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||||
|
block_size=self.kv_cache_spec.block_size,
|
||||||
|
align_chunk_to_block=current_platform.is_cuda(),
|
||||||
|
device=self.device,
|
||||||
|
dcp_world_size=self.dcp_world_size,
|
||||||
|
dcp_local_block_size=self.dcp_local_block_size,
|
||||||
|
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def build(
|
||||||
|
self,
|
||||||
|
common_prefix_len: int,
|
||||||
|
common_attn_metadata: "CommonAttentionMetadata",
|
||||||
|
fast_build: bool = False,
|
||||||
|
) -> T:
|
||||||
|
req_id_per_token = self._build_req_id_per_token(common_attn_metadata)
|
||||||
|
|
||||||
|
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||||
|
common_attn_metadata,
|
||||||
|
decode_threshold=self.reorder_batch_threshold or 1,
|
||||||
|
)
|
||||||
|
(
|
||||||
|
prefill_query_start_loc,
|
||||||
|
prefill_max_query_len,
|
||||||
|
prefill_query_lens_cpu,
|
||||||
|
) = self._build_prefill_fields(common_attn_metadata, num_decodes, num_prefills)
|
||||||
|
|
||||||
|
prefill_max_seq_len = 0
|
||||||
|
prefill: MLACommonPrefillMetadata | None = None
|
||||||
|
if num_prefills > 0 and self._prefill_backend is not None:
|
||||||
|
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||||
|
assert seq_lens_cpu is not None
|
||||||
|
prefill_max_seq_len = int(
|
||||||
|
seq_lens_cpu[num_decodes : num_decodes + num_prefills].max().item()
|
||||||
|
)
|
||||||
|
prefill = MLACommonPrefillMetadata(
|
||||||
|
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
|
||||||
|
query_start_loc=prefill_query_start_loc,
|
||||||
|
max_query_len=prefill_max_query_len,
|
||||||
|
chunked_context=self._build_chunked_context_fields(
|
||||||
|
common_attn_metadata,
|
||||||
|
num_decodes,
|
||||||
|
num_prefills,
|
||||||
|
prefill_query_lens_cpu,
|
||||||
|
),
|
||||||
|
q_data_type=self.model_config.dtype,
|
||||||
|
output_dtype=self.model_config.dtype,
|
||||||
|
prefill_backend=self._prefill_backend,
|
||||||
|
)
|
||||||
|
self._prefill_backend.prepare_metadata(prefill)
|
||||||
|
|
||||||
|
return self.metadata_cls( # type: ignore[call-arg]
|
||||||
|
num_reqs=common_attn_metadata.num_reqs,
|
||||||
|
max_query_len=common_attn_metadata.max_query_len,
|
||||||
|
max_seq_len=common_attn_metadata.max_seq_len,
|
||||||
|
num_actual_tokens=common_attn_metadata.num_actual_tokens,
|
||||||
|
query_start_loc=common_attn_metadata.query_start_loc,
|
||||||
|
slot_mapping=common_attn_metadata.slot_mapping,
|
||||||
|
block_table=common_attn_metadata.block_table_tensor,
|
||||||
|
req_id_per_token=req_id_per_token,
|
||||||
|
seq_lens=common_attn_metadata.seq_lens,
|
||||||
|
block_size=self.kv_cache_spec.block_size,
|
||||||
|
topk_tokens=self.topk_tokens,
|
||||||
|
num_decodes=num_decodes,
|
||||||
|
num_prefills=num_prefills,
|
||||||
|
num_decode_tokens=num_decode_tokens,
|
||||||
|
prefill_max_seq_len=prefill_max_seq_len,
|
||||||
|
prefill=prefill,
|
||||||
|
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_prefill_fields(
|
||||||
|
common_attn_metadata: "CommonAttentionMetadata",
|
||||||
|
num_decodes: int,
|
||||||
|
num_prefills: int,
|
||||||
|
) -> tuple[
|
||||||
|
torch.Tensor | None, # prefill_query_start_loc
|
||||||
|
int, # prefill_max_query_len
|
||||||
|
torch.Tensor | None, # prefill_query_lens_cpu
|
||||||
|
]:
|
||||||
|
if num_prefills == 0:
|
||||||
|
return None, 0, None
|
||||||
|
|
||||||
|
offset = common_attn_metadata.query_start_loc[num_decodes]
|
||||||
|
prefill_query_start_loc = (
|
||||||
|
common_attn_metadata.query_start_loc[num_decodes:] - offset
|
||||||
|
)
|
||||||
|
|
||||||
|
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||||
|
prefill_qsl_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]
|
||||||
|
prefill_query_lens = prefill_qsl_cpu[1:] - prefill_qsl_cpu[:-1]
|
||||||
|
prefill_max_query_len = int(prefill_query_lens.max().item())
|
||||||
|
|
||||||
|
return (
|
||||||
|
prefill_query_start_loc,
|
||||||
|
prefill_max_query_len,
|
||||||
|
prefill_query_lens,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||||
|
"""Sparse MLA base: shared dense-MHA prefill (from MLACommonBaseImpl) plus the
|
||||||
|
sparse top-k MQA decode path. Subclasses implement forward_mqa."""
|
||||||
|
|
||||||
|
is_sparse = True
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
num_heads: int,
|
||||||
|
head_size: int,
|
||||||
|
scale: float,
|
||||||
|
num_kv_heads: int,
|
||||||
|
alibi_slopes: list[float] | None,
|
||||||
|
sliding_window: int | None,
|
||||||
|
kv_cache_dtype: str,
|
||||||
|
logits_soft_cap: float | None,
|
||||||
|
attn_type: str,
|
||||||
|
kv_sharing_target_layer_name: str | None,
|
||||||
|
# MLA-specific
|
||||||
|
q_lora_rank: int | None,
|
||||||
|
kv_lora_rank: int,
|
||||||
|
qk_nope_head_dim: int,
|
||||||
|
qk_rope_head_dim: int,
|
||||||
|
qk_head_dim: int,
|
||||||
|
v_head_dim: int,
|
||||||
|
kv_b_proj: "ColumnParallelLinear",
|
||||||
|
indexer: object | None = None,
|
||||||
|
topk_indices_buffer: torch.Tensor | None = None,
|
||||||
|
q_pad_num_heads: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
scale,
|
||||||
|
num_kv_heads,
|
||||||
|
kv_cache_dtype,
|
||||||
|
kv_lora_rank,
|
||||||
|
qk_nope_head_dim,
|
||||||
|
qk_rope_head_dim,
|
||||||
|
qk_head_dim,
|
||||||
|
v_head_dim,
|
||||||
|
kv_b_proj,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The indexer carries the shared buffer for normal layers and tests;
|
||||||
|
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||||
|
# indexer is not constructed (see deepseek_v2.py).
|
||||||
|
self.topk_indices_buffer: torch.Tensor | None = (
|
||||||
|
indexer.topk_indices_buffer # type: ignore[attr-defined]
|
||||||
|
if indexer is not None
|
||||||
|
else topk_indices_buffer
|
||||||
|
)
|
||||||
|
|
||||||
|
self._use_flashinfer_concat_mla_k = (
|
||||||
|
has_flashinfer()
|
||||||
|
and which("ninja") is not None
|
||||||
|
and (self.num_heads == 128)
|
||||||
|
and (self.qk_nope_head_dim == 128)
|
||||||
|
and (self.qk_rope_head_dim == 64)
|
||||||
|
)
|
||||||
@@ -781,6 +781,10 @@ class AttentionImplBase(ABC, Generic[T]):
|
|||||||
method - subclasses define their own forward interfaces.
|
method - subclasses define their own forward interfaces.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Whether this impl uses a sparse (top-k) attention path. Used by MLA to
|
||||||
|
# route between the dense-MHA prefill and sparse-MQA paths.
|
||||||
|
is_sparse: ClassVar[bool] = False
|
||||||
|
|
||||||
# Required attributes that all impls should have
|
# Required attributes that all impls should have
|
||||||
num_heads: int
|
num_heads: int
|
||||||
head_size: int
|
head_size: int
|
||||||
@@ -976,7 +980,6 @@ class MLAAttentionImpl(AttentionImplBase[T], Generic[T]):
|
|||||||
) -> None:
|
) -> None:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def forward_mha(
|
def forward_mha(
|
||||||
self,
|
self,
|
||||||
q: torch.Tensor,
|
q: torch.Tensor,
|
||||||
@@ -1038,86 +1041,6 @@ class MLAAttentionImpl(AttentionImplBase[T], Generic[T]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class SparseMLAAttentionImpl(AttentionImplBase[T], Generic[T]):
|
|
||||||
"""Sparse MLA attention implementation with only forward_mqa method.
|
|
||||||
|
|
||||||
Sparse MLA implementations only support decode (MQA-style) attention.
|
|
||||||
They do not support prefill (MHA-style) attention.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def fused_output_quant_supported(self, quant_key: "QuantKey"):
|
|
||||||
"""
|
|
||||||
Does this attention implementation support fused output quantization.
|
|
||||||
Since MLA quantization is done manually in forward_impl (common code),
|
|
||||||
all MLA backends support it by default.
|
|
||||||
"""
|
|
||||||
return quant_key in (
|
|
||||||
kFp8StaticTensorSym,
|
|
||||||
kNvfp4Dynamic,
|
|
||||||
kFp8Dynamic128Sym,
|
|
||||||
kFp8Dynamic64Sym,
|
|
||||||
)
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
num_heads: int,
|
|
||||||
head_size: int,
|
|
||||||
scale: float,
|
|
||||||
num_kv_heads: int,
|
|
||||||
alibi_slopes: list[float] | None,
|
|
||||||
sliding_window: int | None,
|
|
||||||
kv_cache_dtype: str,
|
|
||||||
logits_soft_cap: float | None,
|
|
||||||
attn_type: str,
|
|
||||||
kv_sharing_target_layer_name: str | None,
|
|
||||||
# MLA Specific Arguments
|
|
||||||
q_lora_rank: int | None,
|
|
||||||
kv_lora_rank: int,
|
|
||||||
qk_nope_head_dim: int,
|
|
||||||
qk_rope_head_dim: int,
|
|
||||||
qk_head_dim: int,
|
|
||||||
v_head_dim: int,
|
|
||||||
kv_b_proj: "ColumnParallelLinear",
|
|
||||||
indexer: object | None = None,
|
|
||||||
q_pad_num_heads: int | None = None,
|
|
||||||
) -> None:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def forward_mqa(
|
|
||||||
self,
|
|
||||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
||||||
kv_c_and_k_pe_cache: torch.Tensor,
|
|
||||||
attn_metadata: T,
|
|
||||||
layer: AttentionLayer,
|
|
||||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
||||||
"""MQA-style decode forward pass."""
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
def do_kv_cache_update(
|
|
||||||
self,
|
|
||||||
kv_c_normed: torch.Tensor,
|
|
||||||
k_pe: torch.Tensor,
|
|
||||||
kv_cache: torch.Tensor,
|
|
||||||
slot_mapping: torch.Tensor,
|
|
||||||
kv_cache_dtype: str,
|
|
||||||
k_scale: torch.Tensor,
|
|
||||||
) -> None:
|
|
||||||
if kv_cache.numel() == 0:
|
|
||||||
return
|
|
||||||
from vllm import _custom_ops as ops
|
|
||||||
|
|
||||||
ops.concat_and_cache_mla(
|
|
||||||
kv_c_normed,
|
|
||||||
k_pe.squeeze(1),
|
|
||||||
kv_cache,
|
|
||||||
slot_mapping.flatten(),
|
|
||||||
kv_cache_dtype=kv_cache_dtype,
|
|
||||||
scale=k_scale,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def subclass_attention_backend(
|
def subclass_attention_backend(
|
||||||
name_prefix: str,
|
name_prefix: str,
|
||||||
attention_backend_cls: type[AttentionBackend],
|
attention_backend_cls: type[AttentionBackend],
|
||||||
|
|||||||
@@ -4,22 +4,23 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, ClassVar
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.cache import CacheDType
|
from vllm.config.cache import CacheDType
|
||||||
|
from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata
|
||||||
|
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||||
|
SparseMLACommonImpl,
|
||||||
|
SparseMLACommonMetadataBuilder,
|
||||||
|
)
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.utils.torch_utils import np_to_pinned_tensor
|
|
||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
AttentionBackend,
|
AttentionBackend,
|
||||||
AttentionCGSupport,
|
AttentionCGSupport,
|
||||||
AttentionLayer,
|
AttentionLayer,
|
||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
AttentionMetadataBuilder,
|
MLAAttentionImpl,
|
||||||
CommonAttentionMetadata,
|
|
||||||
MultipleOf,
|
MultipleOf,
|
||||||
SparseMLAAttentionImpl,
|
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||||
@@ -50,7 +51,7 @@ class FlashAttnMLASparseBackend(AttentionBackend):
|
|||||||
return FlashAttnMLASparseMetadataBuilder
|
return FlashAttnMLASparseMetadataBuilder
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]:
|
def get_impl_cls() -> type[MLAAttentionImpl[Any]]:
|
||||||
return FlashAttnMLASparseImpl
|
return FlashAttnMLASparseImpl
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -125,13 +126,21 @@ class FlashAttnMLASparseMetadata(AttentionMetadata):
|
|||||||
|
|
||||||
block_table: torch.Tensor
|
block_table: torch.Tensor
|
||||||
req_id_per_token: torch.Tensor
|
req_id_per_token: torch.Tensor
|
||||||
|
seq_lens: torch.Tensor
|
||||||
block_size: int = 64
|
block_size: int = 64
|
||||||
topk_tokens: int = 2048
|
topk_tokens: int = 2048
|
||||||
|
num_decodes: int = 0
|
||||||
|
num_prefills: int = 0
|
||||||
|
num_decode_tokens: int = 0
|
||||||
|
prefill_max_seq_len: int = 0
|
||||||
|
prefill: MLACommonPrefillMetadata | None = None
|
||||||
|
cp_kv_cache_interleave_size: int = 1
|
||||||
|
|
||||||
|
|
||||||
class FlashAttnMLASparseMetadataBuilder(
|
class FlashAttnMLASparseMetadataBuilder(
|
||||||
AttentionMetadataBuilder[FlashAttnMLASparseMetadata]
|
SparseMLACommonMetadataBuilder[FlashAttnMLASparseMetadata]
|
||||||
):
|
):
|
||||||
|
metadata_cls = FlashAttnMLASparseMetadata
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -141,55 +150,16 @@ class FlashAttnMLASparseMetadataBuilder(
|
|||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.vllm_config = vllm_config
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
self.layer_names = layer_names
|
|
||||||
self.kv_cache_spec = kv_cache_spec
|
|
||||||
self.model_config = vllm_config.model_config
|
|
||||||
self.device = device
|
|
||||||
|
|
||||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
num_q_heads = self.model_config.get_num_attention_heads(
|
||||||
|
vllm_config.parallel_config
|
||||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
|
||||||
self.req_id_per_token_buffer = torch.empty(
|
|
||||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
|
||||||
dtype=torch.int32,
|
|
||||||
device=device,
|
|
||||||
)
|
|
||||||
|
|
||||||
def build(
|
|
||||||
self,
|
|
||||||
common_prefix_len: int,
|
|
||||||
common_attn_metadata: CommonAttentionMetadata,
|
|
||||||
fast_build: bool = False,
|
|
||||||
) -> FlashAttnMLASparseMetadata:
|
|
||||||
cm = common_attn_metadata
|
|
||||||
num_tokens = cm.num_actual_tokens
|
|
||||||
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
|
|
||||||
seg_lengths = np.diff(starts)
|
|
||||||
req_id_per_token = np.repeat(
|
|
||||||
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
|
||||||
)
|
|
||||||
|
|
||||||
self.req_id_per_token_buffer.fill_(0)
|
|
||||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
|
||||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
|
||||||
)
|
|
||||||
|
|
||||||
return FlashAttnMLASparseMetadata(
|
|
||||||
num_reqs=cm.num_reqs,
|
|
||||||
max_query_len=cm.max_query_len,
|
|
||||||
max_seq_len=cm.max_seq_len,
|
|
||||||
num_actual_tokens=cm.num_actual_tokens,
|
|
||||||
query_start_loc=cm.query_start_loc,
|
|
||||||
slot_mapping=cm.slot_mapping,
|
|
||||||
block_table=cm.block_table_tensor,
|
|
||||||
req_id_per_token=self.req_id_per_token_buffer[:num_tokens],
|
|
||||||
block_size=self.kv_cache_spec.block_size,
|
|
||||||
topk_tokens=self.topk_tokens,
|
|
||||||
)
|
)
|
||||||
|
threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256)
|
||||||
|
self._init_reorder_batch_threshold(threshold, supports_spec_as_decode=True)
|
||||||
|
|
||||||
|
|
||||||
class FlashAttnMLASparseImpl(SparseMLAAttentionImpl[FlashAttnMLASparseMetadata]):
|
class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_heads: int,
|
num_heads: int,
|
||||||
@@ -217,22 +187,25 @@ class FlashAttnMLASparseImpl(SparseMLAAttentionImpl[FlashAttnMLASparseMetadata])
|
|||||||
"FlashAttnMLASparseImpl currently supports only FP16/BF16 KV cache."
|
"FlashAttnMLASparseImpl currently supports only FP16/BF16 KV cache."
|
||||||
)
|
)
|
||||||
|
|
||||||
self.num_heads = num_heads
|
super().__init__(
|
||||||
self.head_size = head_size
|
num_heads,
|
||||||
self.scale = float(scale)
|
head_size,
|
||||||
self.num_kv_heads = num_kv_heads
|
scale,
|
||||||
self.kv_cache_dtype = kv_cache_dtype
|
num_kv_heads,
|
||||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
alibi_slopes,
|
||||||
self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"]
|
sliding_window,
|
||||||
self.topk_indices_buffer: torch.Tensor | None = (
|
kv_cache_dtype,
|
||||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
logits_soft_cap,
|
||||||
|
attn_type,
|
||||||
|
kv_sharing_target_layer_name,
|
||||||
|
indexer=indexer,
|
||||||
|
topk_indices_buffer=topk_indices_buffer,
|
||||||
|
**mla_args,
|
||||||
)
|
)
|
||||||
assert self.topk_indices_buffer is not None, (
|
assert self.topk_indices_buffer is not None, (
|
||||||
"Indexer or topk_indices_buffer required for sparse MLA"
|
"Indexer or topk_indices_buffer required for sparse MLA"
|
||||||
)
|
)
|
||||||
self.supports_quant_query_input = False
|
self.supports_quant_query_input = False
|
||||||
self.dcp_world_size = -1
|
|
||||||
self.q_pad_num_heads = None
|
|
||||||
|
|
||||||
def forward_mqa(
|
def forward_mqa(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -5,37 +5,33 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, ClassVar
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.cache import CacheDType
|
from vllm.config.cache import CacheDType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention.mla_attention import (
|
from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata
|
||||||
get_mla_dims,
|
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||||
|
SparseMLACommonImpl,
|
||||||
|
SparseMLACommonMetadataBuilder,
|
||||||
)
|
)
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
AttentionBackend,
|
AttentionBackend,
|
||||||
AttentionCGSupport,
|
AttentionCGSupport,
|
||||||
AttentionLayer,
|
AttentionLayer,
|
||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
AttentionMetadataBuilder,
|
|
||||||
AttentionType,
|
AttentionType,
|
||||||
CommonAttentionMetadata,
|
MLAAttentionImpl,
|
||||||
MultipleOf,
|
MultipleOf,
|
||||||
SparseMLAAttentionImpl,
|
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||||
triton_convert_req_index_to_global_index,
|
triton_convert_req_index_to_global_index,
|
||||||
triton_filter_and_convert_dcp_index,
|
triton_filter_and_convert_dcp_index,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.utils import (
|
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||||
KVCacheLayoutType,
|
|
||||||
split_decodes_and_prefills,
|
|
||||||
)
|
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -85,7 +81,7 @@ class FlashInferMLASparseTRTLLMBackend(_FlashInferMLASparseBackendBase):
|
|||||||
return [32, 64]
|
return [32, 64]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_impl_cls() -> type[SparseMLAAttentionImpl]:
|
def get_impl_cls() -> type[MLAAttentionImpl]:
|
||||||
return FlashInferMLASparseImpl
|
return FlashInferMLASparseImpl
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -162,7 +158,7 @@ class FlashInferMLASparseSM120Backend(_FlashInferMLASparseBackendBase):
|
|||||||
return [64, 256]
|
return [64, 256]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_impl_cls() -> type[SparseMLAAttentionImpl]:
|
def get_impl_cls() -> type[MLAAttentionImpl]:
|
||||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
|
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
|
||||||
FlashInferMLASparseSM120Impl,
|
FlashInferMLASparseSM120Impl,
|
||||||
)
|
)
|
||||||
@@ -256,7 +252,10 @@ class FlashInferMLASparseMetadata(AttentionMetadata):
|
|||||||
# Sequence lengths for all requests (context + query)
|
# Sequence lengths for all requests (context + query)
|
||||||
seq_lens: torch.Tensor
|
seq_lens: torch.Tensor
|
||||||
num_decodes: int
|
num_decodes: int
|
||||||
|
num_prefills: int
|
||||||
num_decode_tokens: int
|
num_decode_tokens: int
|
||||||
|
prefill_max_seq_len: int = 0
|
||||||
|
prefill: MLACommonPrefillMetadata | None = None
|
||||||
|
|
||||||
# Sparse-specific
|
# Sparse-specific
|
||||||
block_size: int = 64
|
block_size: int = 64
|
||||||
@@ -265,10 +264,11 @@ class FlashInferMLASparseMetadata(AttentionMetadata):
|
|||||||
|
|
||||||
|
|
||||||
class FlashInferMLASparseMetadataBuilder(
|
class FlashInferMLASparseMetadataBuilder(
|
||||||
AttentionMetadataBuilder[FlashInferMLASparseMetadata]
|
SparseMLACommonMetadataBuilder[FlashInferMLASparseMetadata]
|
||||||
):
|
):
|
||||||
"""Builder for FlashInfer MLA Sparse attention metadata."""
|
"""Builder for FlashInfer MLA Sparse attention metadata."""
|
||||||
|
|
||||||
|
metadata_cls = FlashInferMLASparseMetadata
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -278,75 +278,20 @@ class FlashInferMLASparseMetadataBuilder(
|
|||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.vllm_config = vllm_config
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
self.layer_names = layer_names
|
|
||||||
self.kv_cache_spec = kv_cache_spec
|
|
||||||
self.model_config = vllm_config.model_config
|
|
||||||
self.device = device
|
|
||||||
|
|
||||||
self.mla_dims = get_mla_dims(self.model_config)
|
|
||||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
|
||||||
|
|
||||||
|
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
||||||
|
vllm_config.parallel_config
|
||||||
|
)
|
||||||
|
threshold = {8: 128, 16: 128, 32: 128, 64: 256, 128: 1024}.get(
|
||||||
|
num_q_heads, 1024
|
||||||
|
)
|
||||||
self._init_reorder_batch_threshold(
|
self._init_reorder_batch_threshold(
|
||||||
1,
|
threshold,
|
||||||
supports_spec_as_decode=True,
|
supports_spec_as_decode=True,
|
||||||
supports_dcp_with_varlen=True,
|
supports_dcp_with_varlen=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.req_id_per_token_buffer = torch.empty(
|
|
||||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
|
||||||
dtype=torch.int32,
|
|
||||||
device=device,
|
|
||||||
)
|
|
||||||
|
|
||||||
def build(
|
|
||||||
self,
|
|
||||||
common_prefix_len: int,
|
|
||||||
common_attn_metadata: CommonAttentionMetadata,
|
|
||||||
fast_build: bool = False,
|
|
||||||
) -> FlashInferMLASparseMetadata:
|
|
||||||
cm = common_attn_metadata
|
|
||||||
num_tokens = cm.num_actual_tokens
|
|
||||||
assert self.reorder_batch_threshold is not None
|
|
||||||
num_decodes, _, num_decode_tokens, _ = split_decodes_and_prefills(
|
|
||||||
cm,
|
|
||||||
decode_threshold=self.reorder_batch_threshold,
|
|
||||||
treat_short_extends_as_decodes=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build req_id_per_token mapping
|
|
||||||
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
|
|
||||||
seg_lengths = np.diff(starts)
|
|
||||||
req_id_per_token = np.repeat(
|
|
||||||
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
|
||||||
)
|
|
||||||
|
|
||||||
# Zero-fill for cudagraphs
|
|
||||||
self.req_id_per_token_buffer.fill_(0)
|
|
||||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
|
||||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
|
||||||
)
|
|
||||||
req_id_per_token_tensor = self.req_id_per_token_buffer[:num_tokens]
|
|
||||||
|
|
||||||
return FlashInferMLASparseMetadata(
|
|
||||||
num_reqs=cm.num_reqs,
|
|
||||||
max_query_len=cm.max_query_len,
|
|
||||||
max_seq_len=cm.max_seq_len,
|
|
||||||
num_actual_tokens=cm.num_actual_tokens,
|
|
||||||
query_start_loc=cm.query_start_loc,
|
|
||||||
slot_mapping=cm.slot_mapping,
|
|
||||||
block_table=cm.block_table_tensor,
|
|
||||||
req_id_per_token=req_id_per_token_tensor,
|
|
||||||
seq_lens=cm.seq_lens,
|
|
||||||
num_decodes=num_decodes,
|
|
||||||
num_decode_tokens=num_decode_tokens,
|
|
||||||
block_size=self.kv_cache_spec.block_size,
|
|
||||||
topk_tokens=self.topk_tokens,
|
|
||||||
cp_kv_cache_interleave_size=(
|
|
||||||
self.vllm_config.parallel_config.cp_kv_cache_interleave_size
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# Global workspace buffer (lazily initialized)
|
# Global workspace buffer (lazily initialized)
|
||||||
_fi_sparse_workspace: torch.Tensor | None = None
|
_fi_sparse_workspace: torch.Tensor | None = None
|
||||||
@@ -365,7 +310,7 @@ def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
|
|||||||
return _fi_sparse_workspace
|
return _fi_sparse_workspace
|
||||||
|
|
||||||
|
|
||||||
class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata]):
|
class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||||
"""FlashInfer MLA Sparse implementation.
|
"""FlashInfer MLA Sparse implementation.
|
||||||
|
|
||||||
Uses the TRT-LLM MLA kernel with sparse_mla_top_k parameter for
|
Uses the TRT-LLM MLA kernel with sparse_mla_top_k parameter for
|
||||||
@@ -407,22 +352,20 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
|||||||
"FlashInferMLASparseImpl"
|
"FlashInferMLASparseImpl"
|
||||||
)
|
)
|
||||||
|
|
||||||
self.num_heads = num_heads
|
super().__init__(
|
||||||
self.head_size = head_size
|
num_heads,
|
||||||
self.scale = float(scale)
|
head_size,
|
||||||
self.num_kv_heads = num_kv_heads
|
scale,
|
||||||
self.kv_cache_dtype = kv_cache_dtype
|
num_kv_heads,
|
||||||
|
alibi_slopes,
|
||||||
# MLA-specific dimensions
|
sliding_window,
|
||||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
kv_cache_dtype,
|
||||||
self.qk_nope_head_dim: int = mla_args["qk_nope_head_dim"]
|
logits_soft_cap,
|
||||||
self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"]
|
attn_type,
|
||||||
|
kv_sharing_target_layer_name,
|
||||||
# The indexer carries the shared buffer for normal layers and tests;
|
indexer=indexer,
|
||||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
topk_indices_buffer=topk_indices_buffer,
|
||||||
# indexer is not constructed (see deepseek_v2.py).
|
**mla_args,
|
||||||
self.topk_indices_buffer: torch.Tensor | None = (
|
|
||||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self._workspace_buffer: torch.Tensor | None = None
|
self._workspace_buffer: torch.Tensor | None = None
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import torch
|
|||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
AttentionLayer,
|
AttentionLayer,
|
||||||
AttentionType,
|
AttentionType,
|
||||||
SparseMLAAttentionImpl,
|
MLAAttentionImpl,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||||
FlashInferMLASparseMetadata,
|
FlashInferMLASparseMetadata,
|
||||||
@@ -29,9 +29,11 @@ def _kv_scale_format_for_model(model_type: str | None) -> str:
|
|||||||
return "pow2_fp32"
|
return "pow2_fp32"
|
||||||
|
|
||||||
|
|
||||||
class FlashInferMLASparseSM120Impl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata]):
|
class FlashInferMLASparseSM120Impl(MLAAttentionImpl[FlashInferMLASparseMetadata]):
|
||||||
"""SM120 FlashInfer sparse-MLA implementation."""
|
"""SM120 FlashInfer sparse-MLA implementation."""
|
||||||
|
|
||||||
|
is_sparse = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_heads: int,
|
num_heads: int,
|
||||||
|
|||||||
@@ -1,31 +1,30 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, ClassVar
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import _custom_ops as ops
|
from vllm import _custom_ops as ops
|
||||||
from vllm.config import VllmConfig, get_current_vllm_config
|
from vllm.config import VllmConfig, get_current_vllm_config
|
||||||
from vllm.config.cache import CacheDType
|
from vllm.config.cache import CacheDType
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention.mla_attention import (
|
from vllm.model_executor.layers.attention.mla_attention import MLACommonPrefillMetadata
|
||||||
get_mla_dims,
|
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||||
|
SparseMLACommonImpl,
|
||||||
|
SparseMLACommonMetadataBuilder,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.utils.platform_utils import num_compute_units
|
from vllm.utils.platform_utils import num_compute_units
|
||||||
from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
AttentionBackend,
|
AttentionBackend,
|
||||||
AttentionCGSupport,
|
AttentionCGSupport,
|
||||||
AttentionLayer,
|
AttentionLayer,
|
||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
AttentionMetadataBuilder,
|
|
||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
MultipleOf,
|
MultipleOf,
|
||||||
SparseMLAAttentionImpl,
|
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||||
triton_convert_req_index_to_global_index,
|
triton_convert_req_index_to_global_index,
|
||||||
@@ -109,7 +108,7 @@ class FlashMLASparseBackend(AttentionBackend):
|
|||||||
return FlashMLASparseMetadataBuilder
|
return FlashMLASparseMetadataBuilder
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]:
|
def get_impl_cls() -> type["FlashMLASparseImpl"]:
|
||||||
return FlashMLASparseImpl
|
return FlashMLASparseImpl
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -159,6 +158,14 @@ class FlashMLASparseMetadata(AttentionMetadata):
|
|||||||
block_size: int = 64
|
block_size: int = 64
|
||||||
topk_tokens: int = 2048
|
topk_tokens: int = 2048
|
||||||
|
|
||||||
|
num_decodes: int = 0
|
||||||
|
num_prefills: int = 0
|
||||||
|
num_decode_tokens: int = 0
|
||||||
|
seq_lens: torch.Tensor | None = None
|
||||||
|
prefill_max_seq_len: int = 0
|
||||||
|
prefill: MLACommonPrefillMetadata | None = None
|
||||||
|
cp_kv_cache_interleave_size: int = 1
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class FP8KernelMetadata:
|
class FP8KernelMetadata:
|
||||||
scheduler_metadata: FlashMLASchedMeta
|
scheduler_metadata: FlashMLASchedMeta
|
||||||
@@ -228,8 +235,11 @@ def get_prefill_workspace_size(max_model_len: int):
|
|||||||
return max_model_len * 5
|
return max_model_len * 5
|
||||||
|
|
||||||
|
|
||||||
class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetadata]):
|
class FlashMLASparseMetadataBuilder(
|
||||||
|
SparseMLACommonMetadataBuilder[FlashMLASparseMetadata]
|
||||||
|
):
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
metadata_cls = FlashMLASparseMetadata
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -238,29 +248,30 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad
|
|||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.vllm_config = vllm_config
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
self.layer_names = layer_names
|
|
||||||
cache_config = vllm_config.cache_config
|
cache_config = vllm_config.cache_config
|
||||||
self.kv_cache_spec = kv_cache_spec
|
|
||||||
self.model_config = vllm_config.model_config
|
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
self.device = device
|
|
||||||
|
|
||||||
# Classify single-token queries (plus num_speculative_tokens via
|
num_q_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||||
# supports_spec_as_decode=True) as decodes; longer queries go to
|
if current_platform.is_device_capability_family(100):
|
||||||
# prefill.
|
threshold = {8: 128, 16: 128, 32: 128, 64: 256, 128: 1024}.get(
|
||||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
num_q_heads, 1024
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256)
|
||||||
|
self._init_reorder_batch_threshold(
|
||||||
|
threshold,
|
||||||
|
supports_spec_as_decode=True,
|
||||||
|
)
|
||||||
|
|
||||||
sm_count = num_compute_units(device.index)
|
sm_count = num_compute_units(device.index)
|
||||||
|
|
||||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||||
self.mla_dims = get_mla_dims(self.model_config)
|
|
||||||
# FP8 decode kernel only supports h_q = 64 or 128, so we need to pad
|
# FP8 decode kernel only supports h_q = 64 or 128, so we need to pad
|
||||||
self.fp8_decode_padded_heads = (
|
self.fp8_decode_padded_heads = (
|
||||||
FlashMLASparseImpl._compute_fp8_decode_padded_heads(self.num_heads)
|
FlashMLASparseImpl._compute_fp8_decode_padded_heads(self.num_heads)
|
||||||
)
|
)
|
||||||
|
|
||||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
|
||||||
self.use_fp8_kv_cache = cache_config.cache_dtype == "fp8_ds_mla"
|
self.use_fp8_kv_cache = cache_config.cache_dtype == "fp8_ds_mla"
|
||||||
max_num_seqs = vllm_config.scheduler_config.max_num_seqs
|
max_num_seqs = vllm_config.scheduler_config.max_num_seqs
|
||||||
# Shape: [max_num_seqs], all elements = topk_tokens (constant for full-CG)
|
# Shape: [max_num_seqs], all elements = topk_tokens (constant for full-CG)
|
||||||
@@ -306,11 +317,6 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad
|
|||||||
dtype=torch.int32,
|
dtype=torch.int32,
|
||||||
device=device,
|
device=device,
|
||||||
)
|
)
|
||||||
self.req_id_per_token_buffer = torch.empty(
|
|
||||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
|
||||||
dtype=torch.int32,
|
|
||||||
device=device,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _build_fp8_mixed_decode_prefill(
|
def _build_fp8_mixed_decode_prefill(
|
||||||
self,
|
self,
|
||||||
@@ -493,51 +499,24 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad
|
|||||||
common_attn_metadata: CommonAttentionMetadata,
|
common_attn_metadata: CommonAttentionMetadata,
|
||||||
fast_build: bool = False,
|
fast_build: bool = False,
|
||||||
) -> FlashMLASparseMetadata:
|
) -> FlashMLASparseMetadata:
|
||||||
cm = common_attn_metadata
|
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
|
||||||
num_tokens = cm.num_actual_tokens
|
|
||||||
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
|
|
||||||
seg_lengths = np.diff(starts)
|
|
||||||
req_id_per_token = np.repeat(
|
|
||||||
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
|
||||||
)
|
|
||||||
# Zero-fill for cudagraphs
|
|
||||||
self.req_id_per_token_buffer.fill_(0)
|
|
||||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
|
||||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
|
||||||
)
|
|
||||||
req_id_per_token = self.req_id_per_token_buffer[:num_tokens]
|
|
||||||
|
|
||||||
fp8_extra_metadata: (
|
|
||||||
FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
|
||||||
| FlashMLASparseMetadata.FP8KernelMetadata
|
|
||||||
| None
|
|
||||||
) = None
|
|
||||||
fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
|
fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
|
||||||
|
metadata.fp8_use_mixed_batch = fp8_use_mixed_batch
|
||||||
if self.use_fp8_kv_cache:
|
if self.use_fp8_kv_cache:
|
||||||
if fp8_use_mixed_batch:
|
if fp8_use_mixed_batch:
|
||||||
fp8_extra_metadata = self._build_fp8_mixed_decode_prefill(cm)
|
metadata.fp8_extra_metadata = self._build_fp8_mixed_decode_prefill(
|
||||||
|
common_attn_metadata
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
fp8_extra_metadata = self._build_fp8_separate_prefill_decode(cm)
|
metadata.fp8_extra_metadata = self._build_fp8_separate_prefill_decode(
|
||||||
|
common_attn_metadata
|
||||||
metadata = FlashMLASparseMetadata(
|
)
|
||||||
num_reqs=cm.num_reqs,
|
|
||||||
max_query_len=cm.max_query_len,
|
|
||||||
max_seq_len=cm.max_seq_len,
|
|
||||||
num_actual_tokens=cm.num_actual_tokens,
|
|
||||||
query_start_loc=cm.query_start_loc,
|
|
||||||
slot_mapping=cm.slot_mapping,
|
|
||||||
block_table=cm.block_table_tensor,
|
|
||||||
req_id_per_token=req_id_per_token,
|
|
||||||
block_size=self.kv_cache_spec.block_size,
|
|
||||||
topk_tokens=self.topk_tokens,
|
|
||||||
fp8_extra_metadata=fp8_extra_metadata,
|
|
||||||
fp8_use_mixed_batch=fp8_use_mixed_batch,
|
|
||||||
)
|
|
||||||
|
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]):
|
class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _compute_fp8_decode_padded_heads(num_heads: int) -> int:
|
def _compute_fp8_decode_padded_heads(num_heads: int) -> int:
|
||||||
# FP8 decode kernel only supports h_q = 64 or 128
|
# FP8 decode kernel only supports h_q = 64 or 128
|
||||||
@@ -561,19 +540,22 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]):
|
|||||||
indexer: "Indexer | None" = None,
|
indexer: "Indexer | None" = None,
|
||||||
**mla_args,
|
**mla_args,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.num_heads = num_heads
|
super().__init__(
|
||||||
self.head_size = head_size
|
num_heads,
|
||||||
self.scale = float(scale)
|
head_size,
|
||||||
self.num_kv_heads = num_kv_heads
|
scale,
|
||||||
self.kv_cache_dtype = kv_cache_dtype
|
num_kv_heads,
|
||||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
alibi_slopes,
|
||||||
self.softmax_scale = scale
|
sliding_window,
|
||||||
# The indexer carries the shared buffer for normal layers and tests;
|
kv_cache_dtype,
|
||||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
logits_soft_cap,
|
||||||
# indexer is not constructed (see deepseek_v2.py).
|
attn_type,
|
||||||
self.topk_indices_buffer: torch.Tensor | None = (
|
kv_sharing_target_layer_name,
|
||||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
indexer=indexer,
|
||||||
|
topk_indices_buffer=topk_indices_buffer,
|
||||||
|
**mla_args,
|
||||||
)
|
)
|
||||||
|
self.softmax_scale = scale
|
||||||
# Prefill BF16 kernel requires 64 on Hopper, 128 on Blackwell
|
# Prefill BF16 kernel requires 64 on Hopper, 128 on Blackwell
|
||||||
self.prefill_padding = (
|
self.prefill_padding = (
|
||||||
128 if current_platform.is_device_capability_family(100) else 64
|
128 if current_platform.is_device_capability_family(100) else 64
|
||||||
|
|||||||
@@ -238,7 +238,9 @@ def get_max_prefill_buffer_size(vllm_config: VllmConfig):
|
|||||||
|
|
||||||
|
|
||||||
class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||||
reorder_batch_threshold: int = 1
|
# The indexer opts out of the shared reorder-threshold vote (see __init__),
|
||||||
|
# so this is None; its own split uses self.decode_threshold.
|
||||||
|
reorder_batch_threshold: int | None = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_cudagraph_support(
|
def get_cudagraph_support(
|
||||||
@@ -285,7 +287,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
)
|
)
|
||||||
|
|
||||||
next_n = self.num_speculative_tokens + 1
|
next_n = self.num_speculative_tokens + 1
|
||||||
self.reorder_batch_threshold += self.num_speculative_tokens
|
self.decode_threshold = next_n
|
||||||
|
self.reorder_batch_threshold = None
|
||||||
# NOTE: SM100 datacenter GPUs support any next_n natively via the
|
# NOTE: SM100 datacenter GPUs support any next_n natively via the
|
||||||
# multi-atom paged MQA logits kernels (FP8 and FP4 indexer
|
# multi-atom paged MQA logits kernels (FP8 and FP4 indexer
|
||||||
# caches). Outside the SM100 family the FP8
|
# caches). Outside the SM100 family the FP8
|
||||||
@@ -565,7 +568,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||||
split_decodes_and_prefills(
|
split_decodes_and_prefills(
|
||||||
common_attn_metadata,
|
common_attn_metadata,
|
||||||
decode_threshold=self.reorder_batch_threshold,
|
decode_threshold=self.decode_threshold,
|
||||||
require_uniform=not self.use_flattening,
|
require_uniform=not self.use_flattening,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
"""FlashAttention backend for MLA prefill."""
|
"""FlashAttention backend for MLA prefill."""
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -25,7 +25,10 @@ from vllm.v1.attention.backends.fa_utils import (
|
|||||||
get_flash_attn_version,
|
get_flash_attn_version,
|
||||||
is_flash_attn_varlen_func_available,
|
is_flash_attn_varlen_func_available,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
from vllm.v1.attention.backends.mla.prefill.base import (
|
||||||
|
MLADimensions,
|
||||||
|
MLAPrefillBackend,
|
||||||
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
@@ -40,6 +43,14 @@ else:
|
|||||||
class FlashAttnPrefillBackend(MLAPrefillBackend):
|
class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||||
"""FlashAttention backend for MLA prefill."""
|
"""FlashAttention backend for MLA prefill."""
|
||||||
|
|
||||||
|
supported_mla_dimensions: ClassVar[list[MLADimensions]] = [
|
||||||
|
MLADimensions(
|
||||||
|
qk_nope_head_dim=128,
|
||||||
|
qk_rope_head_dim=64,
|
||||||
|
v_head_dim=128,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_name() -> str:
|
def get_name() -> str:
|
||||||
return "FLASH_ATTN"
|
return "FLASH_ATTN"
|
||||||
@@ -76,7 +87,9 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
|||||||
)
|
)
|
||||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||||
self.flash_attn_varlen_func = flash_attn_varlen_func
|
self.flash_attn_varlen_func = flash_attn_varlen_func
|
||||||
self.vllm_flash_attn_version = get_flash_attn_version(head_size=qk_head_dim)
|
self.vllm_flash_attn_version = get_flash_attn_version(
|
||||||
|
head_size=qk_head_dim, head_size_v=v_head_dim
|
||||||
|
)
|
||||||
if self.vllm_flash_attn_version is not None:
|
if self.vllm_flash_attn_version is not None:
|
||||||
self.flash_attn_varlen_func = functools.partial(
|
self.flash_attn_varlen_func = functools.partial(
|
||||||
flash_attn_varlen_func, fa_version=self.vllm_flash_attn_version
|
flash_attn_varlen_func, fa_version=self.vllm_flash_attn_version
|
||||||
|
|||||||
@@ -24,8 +24,8 @@ from vllm.v1.attention.backend import (
|
|||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
AttentionMetadataBuilder,
|
AttentionMetadataBuilder,
|
||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
|
MLAAttentionImpl,
|
||||||
MultipleOf,
|
MultipleOf,
|
||||||
SparseMLAAttentionImpl,
|
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||||
AiterMLAHelper,
|
AiterMLAHelper,
|
||||||
@@ -626,7 +626,9 @@ def reference_mla_sparse_prefill(
|
|||||||
return (result, lse)
|
return (result, lse)
|
||||||
|
|
||||||
|
|
||||||
class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]):
|
class ROCMAiterMLASparseImpl(MLAAttentionImpl[ROCMAiterMLASparseMetadata]):
|
||||||
|
is_sparse = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_heads: int,
|
num_heads: int,
|
||||||
|
|||||||
@@ -292,8 +292,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
- Chunked prefill (aligns with the indexer's chunking)
|
- Chunked prefill (aligns with the indexer's chunking)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Base threshold: query_len <= 1 is decode
|
reorder_batch_threshold: int | None = None
|
||||||
reorder_batch_threshold: int = 1
|
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
@@ -315,14 +314,13 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
)
|
)
|
||||||
# Decode can have query_len up to
|
# Decode can have query_len up to
|
||||||
# 1 + (2 if parallel drafting else 1) * num_speculative_tokens.
|
# 1 + (2 if parallel drafting else 1) * num_speculative_tokens.
|
||||||
# This MUST match the flashmla_sparse / indexer threshold so that
|
# sparse_swa has no MQA-vs-dense-MHA routing, so multi-token queries take
|
||||||
# all backends agree on the decode/prefill split.
|
# the prefill path and the decode/prefill split stays at that width.
|
||||||
spec_mult = (
|
spec_mult = (
|
||||||
2 if (spec_config is not None and spec_config.parallel_drafting) else 1
|
2 if (spec_config is not None and spec_config.parallel_drafting) else 1
|
||||||
)
|
)
|
||||||
self.decode_threshold = (
|
self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens
|
||||||
self.reorder_batch_threshold + spec_mult * self.num_speculative_tokens
|
self.reorder_batch_threshold = None
|
||||||
)
|
|
||||||
|
|
||||||
hf_config = self.vllm_config.model_config.hf_config
|
hf_config = self.vllm_config.model_config.hf_config
|
||||||
assert hasattr(hf_config, "sliding_window")
|
assert hasattr(hf_config, "sliding_window")
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from vllm.v1.attention.backend import (
|
|||||||
AttentionMetadata,
|
AttentionMetadata,
|
||||||
AttentionMetadataBuilder,
|
AttentionMetadataBuilder,
|
||||||
CommonAttentionMetadata,
|
CommonAttentionMetadata,
|
||||||
SparseMLAAttentionImpl,
|
MLAAttentionImpl,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||||
triton_convert_req_index_to_global_index,
|
triton_convert_req_index_to_global_index,
|
||||||
@@ -170,7 +170,9 @@ class XPUMLASparseMetadataBuilder(AttentionMetadataBuilder[XPUMLASparseMetadata]
|
|||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
class XPUMLASparseImpl(SparseMLAAttentionImpl[XPUMLASparseMetadata]):
|
class XPUMLASparseImpl(MLAAttentionImpl[XPUMLASparseMetadata]):
|
||||||
|
is_sparse = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
num_heads: int,
|
num_heads: int,
|
||||||
|
|||||||
Reference in New Issue
Block a user