[Bugfix][KV Transfer] Reject NixlConnector + expandable_segments:True (#41237)

This commit is contained in:
Roy Wang
2026-05-08 16:47:33 -07:00
committed by GitHub
parent 8bcd8a260c
commit 60851b1d22
2 changed files with 98 additions and 0 deletions
+55
View File
@@ -65,6 +65,61 @@ def test_kv_connector(
assert "existing_key" not in kv_connector_extra_config
def _build_config(
*, kv_connector: str | None, enable_sleep_mode: bool = False
) -> VllmConfig:
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
requiring a real model (avoids HF downloads in CI)."""
from types import SimpleNamespace
kv_transfer_config = (
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
if kv_connector is not None
else None
)
cfg = VllmConfig.__new__(VllmConfig)
cfg.kv_transfer_config = kv_transfer_config
cfg.model_config = SimpleNamespace(enable_sleep_mode=enable_sleep_mode)
cfg._verify_kv_transfer_compat()
return cfg
@pytest.mark.parametrize(
"kv_connector", ["NixlConnector", "MooncakeConnectorV1", "SomeOOTConnector"]
)
def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
"""KV connectors that pin KV cache memory (e.g. via ibv_reg_mr) are
invalidated when expandable_segments lets the CUDA VMM allocator remap
the underlying physical pages. We can't enumerate every connector that
does this (especially OOT ones), so reject the combination whenever any
connector is configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
with pytest.raises(ValueError, match="expandable_segments"):
_build_config(kv_connector=kv_connector)
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812)."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
monkeypatch.setenv(
"PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:512,expandable_segments:False"
)
_build_config(kv_connector="NixlConnector")
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
"""The expandable_segments check only applies when a KV connector is
configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector=None)
def test_kv_offloading_size_only_uses_native_default():
"""Test that setting only kv_offloading_size enables native offloading."""
vllm_config = VllmConfig(