[Bugfix][KV Transfer] Reject NixlConnector + expandable_segments:True (#41237)

This commit is contained in:
Roy Wang
2026-05-08 16:47:33 -07:00
committed by GitHub
parent 8bcd8a260c
commit 60851b1d22
2 changed files with 98 additions and 0 deletions
+55
View File
@@ -65,6 +65,61 @@ def test_kv_connector(
assert "existing_key" not in kv_connector_extra_config
def _build_config(
*, kv_connector: str | None, enable_sleep_mode: bool = False
) -> VllmConfig:
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
requiring a real model (avoids HF downloads in CI)."""
from types import SimpleNamespace
kv_transfer_config = (
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
if kv_connector is not None
else None
)
cfg = VllmConfig.__new__(VllmConfig)
cfg.kv_transfer_config = kv_transfer_config
cfg.model_config = SimpleNamespace(enable_sleep_mode=enable_sleep_mode)
cfg._verify_kv_transfer_compat()
return cfg
@pytest.mark.parametrize(
"kv_connector", ["NixlConnector", "MooncakeConnectorV1", "SomeOOTConnector"]
)
def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
"""KV connectors that pin KV cache memory (e.g. via ibv_reg_mr) are
invalidated when expandable_segments lets the CUDA VMM allocator remap
the underlying physical pages. We can't enumerate every connector that
does this (especially OOT ones), so reject the combination whenever any
connector is configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
with pytest.raises(ValueError, match="expandable_segments"):
_build_config(kv_connector=kv_connector)
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812)."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
monkeypatch.setenv(
"PYTORCH_CUDA_ALLOC_CONF", "max_split_size_mb:512,expandable_segments:False"
)
_build_config(kv_connector="NixlConnector")
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
"""The expandable_segments check only applies when a KV connector is
configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector=None)
def test_kv_offloading_size_only_uses_native_default():
"""Test that setting only kv_offloading_size enables native offloading."""
vllm_config = VllmConfig(
+43
View File
@@ -713,6 +713,48 @@ class VllmConfig:
# This is the same for all backends
self.kv_transfer_config.kv_role = "kv_both"
def _verify_kv_transfer_compat(self) -> None:
"""Reject configurations that silently corrupt KV transfers."""
if (
self.kv_transfer_config is None
or self.kv_transfer_config.kv_connector is None
):
return
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
# remap a virtual address range to different physical pages over the
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
# registrations pointing at stale physical pages after any remap,
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
# NIXL_ERR_REMOTE_DISCONNECT at the first inter-node KV transfer.
# We can't enumerate every in-tree and out-of-tree connector that
# pins memory, so we conservatively reject the combination whenever
# any KV connector is configured.
#
# Sleep mode is exempt: CuMemAllocator.use_memory_pool toggles
# expandable_segments off around its pool (see #40812), so the KV
# cache allocated within that context lands on stable physical pages
# even when the env var is set.
if "expandable_segments:True" not in os.environ.get(
"PYTORCH_CUDA_ALLOC_CONF", ""
):
return
if self.model_config is not None and self.model_config.enable_sleep_mode:
return
raise ValueError(
f"KV connector {self.kv_transfer_config.kv_connector} is "
"incompatible with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True "
"unless enable_sleep_mode is also enabled. PyTorch's CUDA VMM "
"allocator can remap KV cache virtual addresses to different "
"physical pages, invalidating any pinned/registered KV memory "
"(e.g. IB memory regions registered by NIXL or Mooncake). Either "
"unset expandable_segments:True or enable sleep mode (which "
"routes KV allocations through CuMemAllocator's pool, where "
"expandable_segments is automatically disabled)."
)
def __post_init__(self):
"""Verify configs are valid & consistent with each other."""
@@ -1353,6 +1395,7 @@ class VllmConfig:
# Handle the KV connector configs
self._post_init_kv_transfer_config()
self._verify_kv_transfer_compat()
# Log the custom passes that are enabled
self.compilation_config.pass_config.log_enabled_passes()