[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -7,6 +7,7 @@ import pytest
import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.models.deepseek_v32.attention as deepseek_attention
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
@@ -16,6 +17,51 @@ INDEXER_LAYER = "model.layers.0.self_attn.indexer.k_cache"
MLA_LAYER = "model.layers.0.self_attn.attn"
def test_sparse_attention_refreshes_batch_state_inside_eager_segment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
metadata = SimpleNamespace(num_actual_tokens=1)
class BatchStateRefreshed(Exception):
pass
def refresh_batch_state(attn_metadata):
assert attn_metadata is metadata
raise BatchStateRefreshed
layer = SimpleNamespace(
indexer=None,
skip_topk=True,
layer_name=MLA_LAYER,
impl=SimpleNamespace(
prepare_for_batch=refresh_batch_state,
record_logical_topk_ready=lambda: None,
),
)
monkeypatch.setattr(
deepseek_attention,
"get_attention_context",
lambda layer_name: (metadata, None, torch.empty(1), None),
)
with pytest.raises(BatchStateRefreshed):
DeepseekV32Attention._sparse_indexer_and_attn(
layer,
torch.empty(1, dtype=torch.long),
torch.empty(1, 1),
torch.empty(1, 1, 1),
torch.empty(1, 1, 1),
None,
None,
None,
None,
None,
torch.empty(1, 1, 1),
torch.empty(1, 1, 1),
torch.empty(1, 1),
)
def make_indexer_metadata(
*,
num_decodes: int = 0,
@@ -246,6 +292,10 @@ def test_deepseek_v32_dispatches_selected_mha(
_fp8_query=fp8_query,
_use_sparse_mha=lambda _: True,
rotary_emb=lambda _positions, q: (q + 1, None),
impl=SimpleNamespace(
record_logical_topk_ready=lambda: None,
prepare_for_batch=lambda _: None,
),
forward_impl=record_forward_impl,
)
q_nope = torch.randn(2, 1, 2)