[NixlConnector] Initiate deprecation cycle for kv_both role (#43874)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-06-05 11:08:17 +02:00
committed by GitHub
parent e64237ae82
commit d98b8f371c
10 changed files with 94 additions and 24 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ The lease mechanism is controlled through `kv_connector_extra_config` in `--kv-t
vllm serve <MODEL> \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_both",
"kv_role": "kv_producer",
"kv_connector_extra_config": {"kv_lease_duration": 60}
}'
```
+19 -9
View File
@@ -50,7 +50,7 @@ To select a different backend, set `kv_connector_extra_config.backends` in `--kv
vllm serve <MODEL> \
--kv-transfer-config '{
"kv_connector":"NixlConnector",
"kv_role":"kv_both",
"kv_role":"kv_producer",
"kv_connector_extra_config":{"backends":["LIBFABRIC"]}
}'
```
@@ -60,7 +60,7 @@ You can also pass JSON keys individually using dotted arguments, and you can app
```bash
vllm serve <MODEL> \
--kv-transfer-config.kv_connector NixlConnector \
--kv-transfer-config.kv_role kv_both \
--kv-transfer-config.kv_role kv_producer \
--kv-transfer-config.kv_connector_extra_config.backends+ LIBFABRIC
```
@@ -81,7 +81,7 @@ VLLM_NIXL_SIDE_CHANNEL_PORT=5600 \
vllm serve Qwen/Qwen3-0.6B \
--port 8100 \
--enforce-eager \
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail"}'
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail"}'
```
### Consumer (Decoder) Configuration
@@ -96,7 +96,7 @@ VLLM_NIXL_SIDE_CHANNEL_PORT=5601 \
vllm serve Qwen/Qwen3-0.6B \
--port 8200 \
--enforce-eager \
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail"}'
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail"}'
```
### Proxy Server
@@ -212,10 +212,21 @@ sequenceDiagram
Enable bidirectional KV transfer by setting `bidirectional_kv_xfer` in `kv_connector_extra_config` on **both** P and D instances:
```bash
# Prefill instance
vllm serve <MODEL> \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_both",
"kv_role": "kv_producer",
"kv_connector_extra_config": {
"bidirectional_kv_xfer": true
}
}'
# Decode instance
vllm serve <MODEL> \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_consumer",
"kv_connector_extra_config": {
"bidirectional_kv_xfer": true
}
@@ -359,11 +370,10 @@ For multi-host DP deployment, only need to provide the host/port of the head ins
- **kv_producer**: For prefiller instances that generate KV caches
- **kv_consumer**: For decoder instances that consume KV caches from prefiller
- **kv_both**: Enables symmetric functionality where the connector can act as both producer and consumer. This provides flexibility for experimental setups and scenarios where the role distinction is not predetermined.
- **kv_both** (deprecated): Previously used as a catch-all when the role was not predetermined. This value is now deprecated for NixlConnector and will be removed in a future release.
!!! tip
NixlConnector currently does not distinguish `kv_role`; the actual prefiller/decoder roles are determined by the upper-level proxy (e.g., `toy_proxy_server.py` using `--prefiller-hosts` and `--decoder-hosts`).
Therefore, `kv_role` in `--kv-transfer-config` is effectively a placeholder and does not affect NixlConnector's behavior.
!!! warning
`kv_role="kv_both"` is deprecated for NixlConnector. Please set `kv_role="kv_producer"` for prefill instances and `kv_role="kv_consumer"` for decode instances. See [#33702](https://github.com/vllm-project/vllm/issues/33702) for details.
### KV Load Failure Policy
@@ -49,11 +49,13 @@ else
KV_EXTRA_CONFIG=''
fi
# Build the kv-transfer-config once
# Build the kv-transfer-config for P and D
if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then
KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
else
KV_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
fi
# Models to run
@@ -159,7 +161,7 @@ run_tests_for_model() {
--block-size ${PREFILL_BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $PREFILLER_TP_SIZE \
--kv-transfer-config '$KV_CONFIG'"
--kv-transfer-config '$KV_CONFIG_P'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
@@ -207,7 +209,7 @@ run_tests_for_model() {
--enforce-eager \
--block-size ${DECODE_BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--kv-transfer-config '$KV_CONFIG'"
--kv-transfer-config '$KV_CONFIG_D'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
@@ -193,9 +193,11 @@ run_test_for_device() {
local kv_device=$1
if [[ "$kv_device" == "cuda" ]]; then
local kv_config='{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
local kv_config_p='{"kv_connector":"NixlConnector","kv_role":"kv_producer"}'
local kv_config_d='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}'
else
local kv_config="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"${kv_device}\"}"
local kv_config_p="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"${kv_device}\"}"
local kv_config_d="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"${kv_device}\"}"
fi
echo ""
@@ -248,7 +250,7 @@ run_test_for_device() {
--block-size ${BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $PREFILLER_TP_SIZE \
--kv-transfer-config "$kv_config" \
--kv-transfer-config "$kv_config_p" \
--speculative-config "$PREFILL_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND \
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
@@ -287,7 +289,7 @@ run_test_for_device() {
--block-size ${BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $DECODER_TP_SIZE \
--kv-transfer-config "$kv_config" \
--kv-transfer-config "$kv_config_d" \
--speculative-config "$DECODE_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND \
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
@@ -1058,7 +1058,7 @@ def test_multi_connector_mixed_hma_disables_hybrid_kv_cache(monkeypatch):
"connectors": [
{
"kv_connector": "NixlConnector",
"kv_role": "kv_both",
"kv_role": "kv_consumer",
},
{
"kv_connector": "MockConnector",
@@ -1363,7 +1363,7 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
timeout = 6
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_role="kv_consumer",
kv_connector_extra_config={"kv_lease_duration": timeout},
)
llm_kwargs = {
@@ -2737,3 +2737,50 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
f"got {notif!r} (expected {expected_notif!r}, "
f"buggy form would be {bad_notif!r})"
)
def test_kv_both_deprecation_warning(default_vllm_config, dist_init):
"""kv_role='kv_both' should emit a deprecation log warning."""
from unittest.mock import patch
from vllm.logger import _print_warning_once
_print_warning_once.cache_clear()
vllm_config = create_vllm_config(kv_role="kv_both")
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
) as mock_logger:
mock_logger.warning_once = mock_logger.warning_once
NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
)
mock_logger.warning_once.assert_called_once()
msg = mock_logger.warning_once.call_args[0][0]
assert "kv_role='kv_both'" in msg
assert "deprecated" in msg
def test_explicit_kv_role_no_deprecation_warning(default_vllm_config, dist_init):
"""kv_role='kv_consumer' or 'kv_producer' should NOT emit a warning."""
from unittest.mock import patch
for role in ("kv_consumer", "kv_producer"):
vllm_config = create_vllm_config(kv_role=role)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector.logger"
) as mock_logger:
NixlConnector(
vllm_config,
KVConnectorRole.WORKER,
make_kv_cache_config(block_size=16),
)
(
mock_logger.warning_once.assert_not_called(),
(f"kv_role={role!r} should not emit deprecation warning"),
)
@@ -453,7 +453,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):
"""
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_role="kv_consumer",
)
block_size = 16
llm_kwargs = {
@@ -75,7 +75,7 @@ def test_gpu_memory_rixl_hma(model_name, sw_size):
"gpu_memory_utilization": 0.5,
"kv_transfer_config": KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_role="kv_consumer",
),
"max_model_len": 2048,
"disable_hybrid_kv_cache_manager": False,
+1 -1
View File
@@ -103,7 +103,7 @@ def create_vllm_config(
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
kv_connector: str = "NixlConnector",
kv_connector_module_path: str | None = None,
kv_role: str = "kv_both",
kv_role: str = "kv_consumer",
disable_hybrid_kv_cache_manager: bool | None = None,
) -> VllmConfig:
"""Initialize VllmConfig For Testing."""
@@ -94,6 +94,15 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA):
super().__init__(vllm_config, role, kv_cache_config)
assert vllm_config.kv_transfer_config is not None
assert vllm_config.kv_transfer_config.engine_id is not None
if vllm_config.kv_transfer_config.kv_role == "kv_both":
logger.warning_once(
"Using kv_role='kv_both' with NixlConnector is deprecated "
"and will be removed in a future release. Please set "
"kv_role='kv_producer' for prefill instances and "
"kv_role='kv_consumer' for decode instances. "
)
self.kv_cache_config = kv_cache_config
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
self.kv_transfer_config = vllm_config.kv_transfer_config