[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+66 -2
View File
@@ -19,10 +19,12 @@ import vllm.config.vllm as vllm_config_module
import vllm.envs as envs
from vllm.compilation.backends import VllmBackend
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
DeviceConfig,
EngramConfig,
HiSparseConfig,
KernelConfig,
KVTransferConfig,
ModelConfig,
@@ -279,6 +281,62 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
def test_hisparse_requires_v2_model_runner():
config = object.__new__(VllmConfig)
config.attention_config = AttentionConfig(hisparse_config=HiSparseConfig())
with patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", None):
assert config.use_v2_model_runner
with (
patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", False),
pytest.raises(ValueError, match="requires Model Runner V2"),
):
_ = config.use_v2_model_runner
def test_hisparse_rejects_decode_context_parallelism(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
with pytest.raises(ValueError, match="decode context parallelism"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
),
)
def test_hisparse_rejects_pipeline_parallelism(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
with pytest.raises(ValueError, match="pipeline parallelism"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
)
def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
with pytest.raises(ValueError, match="requires the hybrid KV cache manager"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=True,
),
)
def test_hisparse_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
MRV1 paths and off breakable cudagraphs by default."""
@@ -305,7 +363,8 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
# (warning_once args must be hashable for its lru_cache).
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
config = SimpleNamespace(
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]),
attention_config=AttentionConfig(),
)
assert VllmConfig.use_v2_model_runner.fget(config) is False
finally:
@@ -385,6 +444,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
)
config = SimpleNamespace(
model_config=model_config,
attention_config=AttentionConfig(),
speculative_config=SimpleNamespace(method="mtp") if with_mtp else None,
parallel_config=SimpleNamespace(prefill_context_parallel_size=1),
compilation_config=CompilationConfig(
@@ -897,7 +957,10 @@ def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch):
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
config = SimpleNamespace(model_config=model_config)
config = SimpleNamespace(
model_config=model_config,
attention_config=AttentionConfig(),
)
config._get_v2_model_runner_unsupported_features = lambda: []
assert VllmConfig.use_v2_model_runner.fget(config) is expected
@@ -2787,6 +2850,7 @@ def test_draft_sample_method_gumbel_is_rejected():
def _watermarked_vllm_config() -> VllmConfig:
config = object.__new__(VllmConfig)
config.watermark_config = WatermarkConfig(key=42)
config.attention_config = AttentionConfig()
config.speculative_config = None
return config