[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.models.deepseek_v32.attention as deepseek_attention
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
@@ -16,6 +17,51 @@ INDEXER_LAYER = "model.layers.0.self_attn.indexer.k_cache"
|
||||
MLA_LAYER = "model.layers.0.self_attn.attn"
|
||||
|
||||
|
||||
def test_sparse_attention_refreshes_batch_state_inside_eager_segment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
metadata = SimpleNamespace(num_actual_tokens=1)
|
||||
|
||||
class BatchStateRefreshed(Exception):
|
||||
pass
|
||||
|
||||
def refresh_batch_state(attn_metadata):
|
||||
assert attn_metadata is metadata
|
||||
raise BatchStateRefreshed
|
||||
|
||||
layer = SimpleNamespace(
|
||||
indexer=None,
|
||||
skip_topk=True,
|
||||
layer_name=MLA_LAYER,
|
||||
impl=SimpleNamespace(
|
||||
prepare_for_batch=refresh_batch_state,
|
||||
record_logical_topk_ready=lambda: None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deepseek_attention,
|
||||
"get_attention_context",
|
||||
lambda layer_name: (metadata, None, torch.empty(1), None),
|
||||
)
|
||||
|
||||
with pytest.raises(BatchStateRefreshed):
|
||||
DeepseekV32Attention._sparse_indexer_and_attn(
|
||||
layer,
|
||||
torch.empty(1, dtype=torch.long),
|
||||
torch.empty(1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1, 1),
|
||||
torch.empty(1, 1),
|
||||
)
|
||||
|
||||
|
||||
def make_indexer_metadata(
|
||||
*,
|
||||
num_decodes: int = 0,
|
||||
@@ -246,6 +292,10 @@ def test_deepseek_v32_dispatches_selected_mha(
|
||||
_fp8_query=fp8_query,
|
||||
_use_sparse_mha=lambda _: True,
|
||||
rotary_emb=lambda _positions, q: (q + 1, None),
|
||||
impl=SimpleNamespace(
|
||||
record_logical_topk_ready=lambda: None,
|
||||
prepare_for_batch=lambda _: None,
|
||||
),
|
||||
forward_impl=record_forward_impl,
|
||||
)
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from inspect import signature
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
@@ -76,8 +77,16 @@ def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
assert token_counts == (32,)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
@pytest.mark.parametrize("skip_attn", [False, True])
|
||||
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
|
||||
|
||||
runner = _make_runner([])
|
||||
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
|
||||
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
|
||||
None, **kwargs
|
||||
)
|
||||
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
|
||||
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
|
||||
@@ -93,7 +102,7 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
) as get_buckets,
|
||||
patch("vllm.utils.flashinfer.autotune") as autotune,
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
|
||||
assert get_buckets.call_args_list == [call(8192), call(128)]
|
||||
assert autotune.call_args_list == [
|
||||
@@ -106,11 +115,13 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
call(
|
||||
num_tokens=128,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user