[Bugfix] adjust_dcp_kv_cache_interleave_size for NixlConnector only (#54803)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-02 12:31:14 +02:00
committed by GitHub
co-authored by cjackal
parent 3140531773
commit f81eb41934
3 changed files with 91 additions and 22 deletions
+74 -5
View File
@@ -114,7 +114,34 @@ def test_per_request_spec_decode_metrics_requires_spec_decode():
)
def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
@pytest.mark.parametrize(
"kv_transfer_config",
[
KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
),
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
{
"kv_connector": "NixlConnector",
"kv_role": "kv_both",
},
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
},
]
},
),
],
)
def test_pd_dcp_interleave_size_is_adjusted_to_block_size(
caplog, disable_log_dedup, kv_transfer_config
):
config = VllmConfig(
cache_config=CacheConfig(block_size=16),
device_config=DeviceConfig(device="cpu"),
@@ -124,10 +151,7 @@ def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
cp_kv_cache_interleave_size=3,
distributed_executor_backend="mp",
),
kv_transfer_config=KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
),
kv_transfer_config=kv_transfer_config,
)
kv_cache_config = SimpleNamespace(
@@ -140,6 +164,51 @@ def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
assert "automatically adjusted from 3 to block_size 16" in caplog.text
def test_kv_offloading_does_not_adjust_dcp_interleave_size():
config = VllmConfig(
cache_config=CacheConfig(block_size=16),
device_config=DeviceConfig(device="cpu"),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
),
kv_transfer_config=KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
),
)
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))]
)
config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
assert config.parallel_config.cp_kv_cache_interleave_size == 1
def test_kv_offloading_does_not_skip_dcp_interleave_validation():
config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
mamba_cache_mode="none",
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=2,
cp_kv_cache_interleave_size=3,
),
scheduler_config=SimpleNamespace(disable_chunked_mm_input=False),
kv_transfer_config=KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
),
)
with pytest.raises(AssertionError, match="divisible by"):
VllmConfig.validate_block_size(config)
def test_compile_config_repr_succeeds():
# setup: VllmBackend mutates the config object
config = VllmConfig()
+16 -16
View File
@@ -2746,7 +2746,7 @@ class VllmConfig:
def adjust_dcp_kv_cache_interleave_size(
self, kv_cache_config: "KVCacheConfig"
) -> None:
"""Normalize DCP interleave size against the resolved block_size for PD.
"""Normalize DCP interleave size against block_size for NIXL P/D.
Called by each worker (via ensure_kv_transfer_initialized), once it knows its
own final block_size via kv_cache_config.
@@ -2754,11 +2754,6 @@ class VllmConfig:
dcp_size = self.parallel_config.decode_context_parallel_size
if dcp_size <= 1:
return
# Get the kernel block_size, but don't use resolve_kv_cache_block_size to avoid
# scaling by dcp_size (we need the local block_size here).
local_block_size = min(
g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups
)
if self.parallel_config.dcp_kv_cache_interleave_size > 1 and (
self.parallel_config.cp_kv_cache_interleave_size
!= self.parallel_config.dcp_kv_cache_interleave_size
@@ -2772,11 +2767,17 @@ class VllmConfig:
"deprecated when PCP is fully supported."
)
if (
self.kv_transfer_config is not None
and self.kv_transfer_config.kv_connector is not None
and self.parallel_config.cp_kv_cache_interleave_size != local_block_size
if self.kv_transfer_config is None or not self.kv_transfer_config.has_connector(
"NixlConnector"
):
return
# Get the kernel block_size, but don't use resolve_kv_cache_block_size to avoid
# scaling by dcp_size (we need the local block_size here).
local_block_size = min(
g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups
)
if self.parallel_config.cp_kv_cache_interleave_size != local_block_size:
interleave = self.parallel_config.cp_kv_cache_interleave_size
self.parallel_config.cp_kv_cache_interleave_size = local_block_size
logger.info_once(
@@ -2797,14 +2798,13 @@ class VllmConfig:
"""
block_size = self.cache_config.block_size
# Skip DCP interleave-size compatibility when a KV connector is configured:
# cp_kv_cache_interleave_size is pinned to block_size for PD by each worker
pd_active = (
# Skip DCP interleave-size compatibility for NIXL P/D: the interleave
# size is pinned to block_size by each worker.
nixl_pd_active = (
self.kv_transfer_config is not None
and self.kv_transfer_config.kv_connector is not None
and self.kv_transfer_config.is_kv_transfer_instance
and self.kv_transfer_config.has_connector("NixlConnector")
)
if self.parallel_config.decode_context_parallel_size > 1 and not pd_active:
if self.parallel_config.decode_context_parallel_size > 1 and not nixl_pd_active:
assert (
self.parallel_config.cp_kv_cache_interleave_size <= block_size
and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0
@@ -85,7 +85,7 @@ def ensure_kv_transfer_initialized(
vllm_config.kv_transfer_config.is_kv_transfer_instance
and _KV_CONNECTOR_AGENT is None
):
# PD only supports an interleave_size equal to block_size.
# NIXL P/D requires an interleave_size equal to block_size.
vllm_config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
_sync_engine_id_across_tp(vllm_config)