[Config] Infer HiSparse attention config from HiSparseConnector (#57041)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -337,6 +337,108 @@ def test_hisparse_rejects_non_cuda(monkeypatch):
|
||||
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_transfer_config",
|
||||
[
|
||||
KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
),
|
||||
KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "OffloadingConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
|
||||
},
|
||||
{
|
||||
"kv_connector": "HiSparseConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"host_pool_gib": 128},
|
||||
},
|
||||
]
|
||||
},
|
||||
),
|
||||
KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_consumer",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
{"kv_connector": "NixlConnector", "kv_role": "kv_consumer"},
|
||||
{
|
||||
"kv_connector": "HiSparseConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"host_pool_gib": 128},
|
||||
},
|
||||
]
|
||||
},
|
||||
),
|
||||
],
|
||||
ids=["standalone", "multi-connector", "pd-decode"],
|
||||
)
|
||||
def test_hisparse_connector_implies_attention_config(monkeypatch, kv_transfer_config):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
|
||||
config = VllmConfig(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
# Skip the HMA auto-detect block: it imports the connector class,
|
||||
# which pulls the platform's compiled attention extensions.
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
),
|
||||
)
|
||||
assert isinstance(config.attention_config.hisparse_config, HiSparseConfig)
|
||||
|
||||
|
||||
def test_hisparse_connector_preserves_explicit_attention_config(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
|
||||
config = VllmConfig(
|
||||
attention_config=AttentionConfig(
|
||||
hisparse_config=HiSparseConfig(device_buffer_size=512)
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
),
|
||||
)
|
||||
assert config.attention_config.hisparse_config.device_buffer_size == 512
|
||||
|
||||
|
||||
def test_hisparse_connector_without_cuda_still_rejected(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
|
||||
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
|
||||
VllmConfig(
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_no_hisparse_connector_keeps_attention_config_unset(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
config = VllmConfig(
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="OffloadingConnector", kv_role="kv_both"
|
||||
)
|
||||
)
|
||||
assert config.attention_config.hisparse_config is None
|
||||
|
||||
|
||||
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
|
||||
MRV1 paths and off breakable cudagraphs by default."""
|
||||
|
||||
@@ -88,7 +88,9 @@ class AttentionConfig:
|
||||
|
||||
hisparse_config: HiSparseConfig | None = None
|
||||
"""HiSparse host-resident KV configuration. Setting this enables experimental
|
||||
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering."""
|
||||
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
|
||||
with defaults when HiSparseConnector is configured (directly or via
|
||||
MultiConnector); set it explicitly only to tune its fields."""
|
||||
|
||||
use_non_causal: bool = False
|
||||
"""Whether to use non-causal (bidirectional) attention."""
|
||||
|
||||
+8
-1
@@ -27,7 +27,7 @@ from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
from .attention import AttentionConfig
|
||||
from .attention import AttentionConfig, HiSparseConfig
|
||||
from .cache import CacheConfig
|
||||
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
from .device import DeviceConfig
|
||||
@@ -1630,6 +1630,13 @@ class VllmConfig:
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
self.attention_config.hisparse_config is None
|
||||
and self.kv_transfer_config is not None
|
||||
and self.kv_transfer_config.has_connector("HiSparseConnector")
|
||||
):
|
||||
self.attention_config.hisparse_config = HiSparseConfig()
|
||||
|
||||
if self.attention_config.hisparse_config is not None:
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("HiSparse currently requires NVIDIA CUDA.")
|
||||
|
||||
Reference in New Issue
Block a user