[Bugfix][KV Transfer] Reject NixlConnector + expandable_segments:True (#41237)
This commit is contained in:
@@ -65,6 +65,61 @@ def test_kv_connector(
|
||||
assert "existing_key" not in kv_connector_extra_config
|
||||
|
||||
|
||||
def _build_config(
|
||||
*, kv_connector: str | None, enable_sleep_mode: bool = False
|
||||
) -> VllmConfig:
|
||||
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
|
||||
requiring a real model (avoids HF downloads in CI)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
kv_transfer_config = (
|
||||
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
|
||||
if kv_connector is not None
|
||||
else None
|
||||
)
|
||||
cfg = VllmConfig.__new__(VllmConfig)
|
||||
cfg.kv_transfer_config = kv_transfer_config
|
||||
cfg.model_config = SimpleNamespace(enable_sleep_mode=enable_sleep_mode)
|
||||
cfg._verify_kv_transfer_compat()
|
||||
return cfg
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_connector", ["NixlConnector", "MooncakeConnectorV1", "SomeOOTConnector"]
|
||||
)
|
||||
def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
|
||||
"""KV connectors that pin KV cache memory (e.g. via ibv_reg_mr) are
|
||||
invalidated when expandable_segments lets the CUDA VMM allocator remap
|
||||
the underlying physical pages. We can't enumerate every connector that
|
||||
does this (especially OOT ones), so reject the combination whenever any
|
||||
connector is configured."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
with pytest.raises(ValueError, match="expandable_segments"):
|
||||
_build_config(kv_connector=kv_connector)
|
||||
|
||||
|
||||
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
|
||||
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
|
||||
auto-disables expandable_segments (see #40812)."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
|
||||
|
||||
|
||||
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
|
||||
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
|
||||
monkeypatch.setenv(
|
||||
"PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:512,expandable_segments:False"
|
||||
)
|
||||
_build_config(kv_connector="NixlConnector")
|
||||
|
||||
|
||||
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
|
||||
"""The expandable_segments check only applies when a KV connector is
|
||||
configured."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector=None)
|
||||
|
||||
|
||||
def test_kv_offloading_size_only_uses_native_default():
|
||||
"""Test that setting only kv_offloading_size enables native offloading."""
|
||||
vllm_config = VllmConfig(
|
||||
|
||||
Reference in New Issue
Block a user