[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -47,6 +47,35 @@ _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
)
tuned: set[tuple[object, ...]] = set()
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
impl = getattr(layer, "impl", None)
if not isinstance(impl, FlashInferMLASparseImpl):
continue
if getattr(layer, "hisparse_cache", None) is None:
continue
if impl.topk_indices_buffer is None:
continue
key = (
impl.kv_cache_dtype,
impl.num_heads,
impl.qk_nope_head_dim,
impl.qk_rope_head_dim,
impl.kv_lora_rank,
impl.topk_indices_buffer.shape[1],
)
if key in tuned:
continue
impl.autotune_hisparse_decode(layer)
tuned.add(key)
def _attention_backend_name(backend: object) -> str | None:
get_name = getattr(backend, "get_name", None)
if get_name is None:
+14 -2
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
write_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
deepseek_v4_sparse_mla_attention_warmup,
flashinfer_sparse_mla_decode_autotune_warmup,
)
@@ -353,9 +354,12 @@ def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ..
return tuple(dict.fromkeys(token_counts))
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
def _run_flashinfer_autotune_dummy_runs(
runner: "GPUModelRunner", *, skip_attn: bool = False
) -> None:
import vllm.utils.flashinfer as fi_utils
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
for num_tokens in _flashinfer_autotune_token_counts(runner):
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
logger.info(
@@ -369,6 +373,7 @@ def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**dummy_run_kwargs,
)
@@ -431,7 +436,14 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
_run_flashinfer_autotune_dummy_runs(runner)
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally: