[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -25,6 +25,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
@@ -353,9 +354,12 @@ def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ..
|
||||
return tuple(dict.fromkeys(token_counts))
|
||||
|
||||
|
||||
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
def _run_flashinfer_autotune_dummy_runs(
|
||||
runner: "GPUModelRunner", *, skip_attn: bool = False
|
||||
) -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
|
||||
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner):
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
|
||||
logger.info(
|
||||
@@ -369,6 +373,7 @@ def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**dummy_run_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -431,7 +436,14 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user