[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -47,6 +47,35 @@ _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
|
||||
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
|
||||
|
||||
|
||||
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
|
||||
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
)
|
||||
|
||||
tuned: set[tuple[object, ...]] = set()
|
||||
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
|
||||
impl = getattr(layer, "impl", None)
|
||||
if not isinstance(impl, FlashInferMLASparseImpl):
|
||||
continue
|
||||
if getattr(layer, "hisparse_cache", None) is None:
|
||||
continue
|
||||
if impl.topk_indices_buffer is None:
|
||||
continue
|
||||
key = (
|
||||
impl.kv_cache_dtype,
|
||||
impl.num_heads,
|
||||
impl.qk_nope_head_dim,
|
||||
impl.qk_rope_head_dim,
|
||||
impl.kv_lora_rank,
|
||||
impl.topk_indices_buffer.shape[1],
|
||||
)
|
||||
if key in tuned:
|
||||
continue
|
||||
impl.autotune_hisparse_decode(layer)
|
||||
tuned.add(key)
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
get_name = getattr(backend, "get_name", None)
|
||||
if get_name is None:
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
@@ -353,9 +354,12 @@ def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ..
|
||||
return tuple(dict.fromkeys(token_counts))
|
||||
|
||||
|
||||
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
def _run_flashinfer_autotune_dummy_runs(
|
||||
runner: "GPUModelRunner", *, skip_attn: bool = False
|
||||
) -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
|
||||
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner):
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
|
||||
logger.info(
|
||||
@@ -369,6 +373,7 @@ def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**dummy_run_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -431,7 +436,14 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user