[Config] Infer HiSparse attention config from HiSparseConnector (#57041)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-15 19:44:26 +00:00
committed by GitHub
parent 2fcc524788
commit df42d112ee
3 changed files with 113 additions and 2 deletions
+102
View File
@@ -337,6 +337,108 @@ def test_hisparse_rejects_non_cuda(monkeypatch):
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
@pytest.mark.parametrize(
"kv_transfer_config",
[
KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
),
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
{
"kv_connector": "HiSparseConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"host_pool_gib": 128},
},
]
},
),
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_consumer",
kv_connector_extra_config={
"connectors": [
{"kv_connector": "NixlConnector", "kv_role": "kv_consumer"},
{
"kv_connector": "HiSparseConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"host_pool_gib": 128},
},
]
},
),
],
ids=["standalone", "multi-connector", "pd-decode"],
)
def test_hisparse_connector_implies_attention_config(monkeypatch, kv_transfer_config):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
config = VllmConfig(
kv_transfer_config=kv_transfer_config,
# Skip the HMA auto-detect block: it imports the connector class,
# which pulls the platform's compiled attention extensions.
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=False,
),
)
assert isinstance(config.attention_config.hisparse_config, HiSparseConfig)
def test_hisparse_connector_preserves_explicit_attention_config(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
config = VllmConfig(
attention_config=AttentionConfig(
hisparse_config=HiSparseConfig(device_buffer_size=512)
),
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=False,
),
)
assert config.attention_config.hisparse_config.device_buffer_size == 512
def test_hisparse_connector_without_cuda_still_rejected(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
VllmConfig(
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
)
)
def test_no_hisparse_connector_keeps_attention_config_unset(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
config = VllmConfig(
kv_transfer_config=KVTransferConfig(
kv_connector="OffloadingConnector", kv_role="kv_both"
)
)
assert config.attention_config.hisparse_config is None
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
MRV1 paths and off breakable cudagraphs by default."""
+3 -1
View File
@@ -88,7 +88,9 @@ class AttentionConfig:
hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering."""
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
with defaults when HiSparseConnector is configured (directly or via
MultiConnector); set it explicitly only to tune its fields."""
use_non_causal: bool = False
"""Whether to use non-causal (bidirectional) attention."""
+8 -1
View File
@@ -27,7 +27,7 @@ from vllm.triton_utils import HAS_TRITON
from vllm.utils import random_uuid
from vllm.utils.hashing import safe_hash
from .attention import AttentionConfig
from .attention import AttentionConfig, HiSparseConfig
from .cache import CacheConfig
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
from .device import DeviceConfig
@@ -1630,6 +1630,13 @@ class VllmConfig:
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_override_dynamic_sd_cudagraph_mode()
if (
self.attention_config.hisparse_config is None
and self.kv_transfer_config is not None
and self.kv_transfer_config.has_connector("HiSparseConnector")
):
self.attention_config.hisparse_config = HiSparseConfig()
if self.attention_config.hisparse_config is not None:
if not current_platform.is_cuda():
raise ValueError("HiSparse currently requires NVIDIA CUDA.")