[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from inspect import signature
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
@@ -76,8 +77,16 @@ def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
assert token_counts == (32,)
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
@pytest.mark.parametrize("skip_attn", [False, True])
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
runner = _make_runner([])
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
None, **kwargs
)
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
@@ -93,7 +102,7 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
) as get_buckets,
patch("vllm.utils.flashinfer.autotune") as autotune,
):
_run_flashinfer_autotune_dummy_runs(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
assert get_buckets.call_args_list == [call(8192), call(128)]
assert autotune.call_args_list == [
@@ -106,11 +115,13 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
call(
num_tokens=128,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
]