[Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: mukeshbhandarkar <shell00035@gmail.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
hangangqiang
2026-09-18 18:11:05 -07:00
committed by GitHub
co-authored by mukeshbhandarkar aoshen02 Claude Fable 5.1 aoshen02
parent 729ebac498
commit 90531daf6d
11 changed files with 282 additions and 14 deletions
+2
View File
@@ -6,6 +6,7 @@ mod cache;
mod collective_rpc;
mod health;
mod inference;
mod kv_event_sources;
mod load;
mod lora;
mod metrics;
@@ -91,6 +92,7 @@ fn build_router_with_options(
.route("/metrics", get(metrics::scrape))
.route("/load", get(load::load))
.route("/version", get(version::version))
.route("/kv_event_sources", get(kv_event_sources::kv_event_sources))
// OpenAI-compatible endpoints
.route("/v1/models", get(openai::list_models))
.route("/v1/completions", post(openai::completions))
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use std::sync::Arc;
use axum::Json;
use axum::extract::State;
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
use crate::state::AppState;
/// KV-event publisher config of each data-parallel rank, keyed by rank.
pub async fn kv_event_sources(
State(state): State<Arc<AppState>>,
) -> Json<BTreeMap<u32, KvEventsConfig>> {
Json(
state
.engine_core_client()
.ready_responses()
.into_iter()
.filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?)))
.collect(),
)
}
+50
View File
@@ -2054,6 +2054,56 @@ async fn version_returns_engine_vllm_version() {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn kv_event_sources_reports_each_ready_engine() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
data_parallel_rank: 1,
kv_events_config: Some(
vllm_engine_core_client::protocol::handshake::KvEventsConfig {
enable_kv_cache_events: true,
publisher: "zmq".to_string(),
endpoint: "tcp://10.0.0.2:41233".to_string(),
replay_endpoint: None,
buffer_steps: 10_000,
hwm: 100_000,
max_queue_size: 100_000,
topic: "kv".to_string(),
},
),
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/kv_event_sources")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(
json,
json!({
"1": {
"enable_kv_cache_events": true,
"publisher": "zmq",
"endpoint": "tcp://10.0.0.2:41233",
"replay_endpoint": null,
"buffer_steps": 10000,
"hwm": 100000,
"max_queue_size": 100000,
"topic": "kv",
}
})
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn server_info_endpoint_is_dev_mode_only() {
+4
View File
@@ -91,8 +91,11 @@ class MockSubscriber:
if isinstance(replay_endpoints, str):
replay_endpoints = [replay_endpoints]
ipv6 = any("[" in ep for ep in pub_endpoints + (replay_endpoints or []))
# Set up subscriber socket - connect to all endpoints
self.sub = self.ctx.socket(zmq.SUB)
self.sub.setsockopt(zmq.IPV6, ipv6)
self.sub.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8"))
for endpoint in pub_endpoints:
self.sub.connect(endpoint)
@@ -103,6 +106,7 @@ class MockSubscriber:
if replay_endpoints:
for replay_endpoint in replay_endpoints:
replay = self.ctx.socket(zmq.DEALER)
replay.setsockopt(zmq.IPV6, ipv6)
replay.connect(replay_endpoint)
self.replay_sockets.append(replay)
+74
View File
@@ -332,3 +332,77 @@ def test_event_publisher_factory(random_port):
publisher = EventPublisherFactory.create(config, DP_RANK)
assert isinstance(publisher, ZmqEventPublisher)
publisher.shutdown()
@pytest.mark.parametrize(
("endpoint", "expected"),
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
)
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
"""Port 0 is not offset: every rank asks the OS for its own port."""
from vllm.distributed.kv_events import ZmqEventPublisher
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
node's IP and report endpoints a subscriber can use for events and replay."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
from vllm.utils.network_utils import split_zmq_path
from .conftest import MockSubscriber
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint="tcp://*:0",
replay_endpoint="tcp://*:0",
topic="kv",
)
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
try:
resolved = [pub.get_publisher_config() for pub in pubs]
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
assert {h for h, _ in hosts_ports} == {node_ip}
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
assert config.endpoint == "tcp://*:0"
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
try:
# PUB/SUB drops messages sent before the subscription lands, so
# keep publishing until both ranks are heard.
ranks: set[int] = set()
deadline = time.time() + 10
while ranks != {0, 1} and time.time() < deadline:
for pub in pubs:
pub.publish(create_test_events(1))
while (result := sub.receive_one(timeout=100)) is not None:
ranks.add(result[1].data_parallel_rank)
assert ranks == {0, 1}
sub.request_replay(0, socket_idx=1)
assert sub.receive_replay(socket_idx=1)
finally:
sub.close()
finally:
for pub in pubs:
pub.shutdown()
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
publisher refuses instead of reporting a wildcard."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
)
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
EventPublisherFactory.create(config, DP_RANK)
@@ -214,6 +214,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
client.eng_start_index = 0
client._kv_event_sources = {}
return client
@@ -1490,3 +1491,57 @@ def test_ready_response_queries_executor_with_weight_transfer():
executor.supports_draft_weight_updates.assert_called_once()
assert response.supports_draft_weight_updates is True
def test_apply_ready_response_retains_kv_event_sources():
"""Each engine's resolved publisher config is kept by DP rank; engines
without KV events contribute nothing."""
import msgspec
from vllm.config.kv_events import KVEventsConfig
client = object.__new__(MPClient)
client._effective_attention_block_sizes = set()
client._kv_event_sources = {}
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
)
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
return msgspec.msgpack.encode(
EngineCoreReadyResponse(
max_model_len=8192,
num_gpu_blocks=100,
block_size=16,
dp_stats_address=None,
dtype="bfloat16",
vllm_version="test",
world_size=2,
data_parallel_size=2,
tensor_parallel_size=1,
pipeline_parallel_size=1,
decode_context_parallel_size=1,
data_parallel_rank=rank,
max_num_seqs=256,
max_num_batched_tokens=8192,
instance_id="test-instance",
supports_lora=False,
max_loras=0,
kv_events_config=config,
)
)
rank1 = KVEventsConfig(
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:41233"
)
client._apply_ready_response(ready(1, rank1))
client._apply_ready_response(ready(0, None))
assert client.get_kv_event_sources() == {1: rank1}
# A rank that comes back (elastic re-add) reports its new port.
rank1_again = KVEventsConfig(
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:47001"
)
client._apply_ready_response(ready(1, rank1_again))
assert client.get_kv_event_sources() == {1: rank1_again}
+40 -14
View File
@@ -17,6 +17,12 @@ import zmq
from vllm.config.kv_events import KVEventsConfig
from vllm.logger import init_logger
from vllm.utils.network_utils import (
get_ip,
get_tcp_uri,
is_valid_ipv6_address,
split_zmq_path,
)
from vllm.v1.core.kv_cache_utils import ExternalBlockHash
logger = init_logger(__name__)
@@ -349,6 +355,8 @@ class ZmqEventPublisher(EventPublisher):
replay_endpoint, self._dp_rank
)
assert self._endpoint is not None
self._hwm = hwm
self._socket_setup()
self._publisher_config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
@@ -359,8 +367,6 @@ class ZmqEventPublisher(EventPublisher):
max_queue_size=max_queue_size,
topic=topic,
)
self._hwm = hwm
self._socket_setup()
# Payload
self._seq_gen = count()
@@ -423,16 +429,10 @@ class ZmqEventPublisher(EventPublisher):
if self._pub is None:
self._pub = self._ctx.socket(zmq.PUB)
self._pub.set_hwm(self._hwm)
# Heuristic: bind if wildcard / * present, else connect.
# bind stable, connect volatile convention
if self._endpoint is not None and (
"*" in self._endpoint
or "::" in self._endpoint
or self._endpoint.startswith("ipc://")
or self._endpoint.startswith("inproc://")
):
self._pub.bind(self._endpoint)
elif self._endpoint is not None:
assert self._endpoint is not None
if self._is_bind_endpoint(self._endpoint):
self._endpoint = self._bind(self._pub, self._endpoint)
else:
self._pub.connect(self._endpoint)
# Set up replay socket: use ROUTER
@@ -441,7 +441,31 @@ class ZmqEventPublisher(EventPublisher):
# 3) works in our non‑blocking poll loop alongside PUB
if self._replay_endpoint is not None:
self._replay = self._ctx.socket(zmq.ROUTER)
self._replay.bind(self._replay_endpoint)
self._replay_endpoint = self._bind(self._replay, self._replay_endpoint)
@staticmethod
def _is_bind_endpoint(endpoint: str) -> bool:
"""Bind wildcard and ipc/inproc endpoints; connect to the rest."""
return (
"*" in endpoint
or "::" in endpoint
or endpoint.startswith(("ipc://", "inproc://"))
)
@staticmethod
def _bind(sock: zmq.Socket, endpoint: str) -> str:
"""Bind; tcp://*:0 binds this node's IP and returns the bound endpoint."""
if endpoint != "tcp://*:0":
sock.bind(endpoint)
return endpoint
ip = get_ip()
if ip in ("0.0.0.0", "::"):
raise ValueError("tcp://*:0 needs a node IP; set VLLM_HOST_IP")
if is_valid_ipv6_address(ip):
sock.setsockopt(zmq.IPV6, 1)
sock.bind(get_tcp_uri(ip, 0))
_, _, port = split_zmq_path(sock.getsockopt_string(zmq.LAST_ENDPOINT))
return get_tcp_uri(ip, int(port))
def _publisher_thread(self) -> None:
"""Background thread that processes the event queue."""
@@ -514,7 +538,7 @@ class ZmqEventPublisher(EventPublisher):
Returns:
The endpoint with the port offset by data_parallel_rank
or suffix appended
or suffix appended; a tcp port of 0 is left for the OS.
"""
# Do nothing if input is None or data_parallel_rank is 0
@@ -529,6 +553,8 @@ class ZmqEventPublisher(EventPublisher):
last_colon_idx = endpoint.rfind(":")
base_addr = endpoint[:last_colon_idx]
base_port = int(endpoint[last_colon_idx + 1 :])
if base_port == 0:
return endpoint
new_port = base_port + data_parallel_rank
return f"{base_addr}:{new_port}"
return endpoint
+5
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from vllm.config import ModelConfig, VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
@@ -62,6 +63,10 @@ class EngineClient(ABC):
@abstractmethod
def dead_error(self) -> BaseException: ...
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
"""KV-event publisher config of each engine, keyed by DP rank."""
return {}
def check_admission( # noqa: B027
self, n: int = 1, request_id: str | None = None
) -> None:
@@ -54,3 +54,10 @@ async def get_server_load_metrics(request: Request):
async def show_version():
ver = {"version": VLLM_VERSION}
return JSONResponse(content=ver)
@router.get("/kv_event_sources")
async def kv_event_sources(raw_request: Request):
"""KV-event publisher config per DP rank; empty on render-only servers."""
client: EngineClient | None = raw_request.app.state.engine_client
return client.get_kv_event_sources() if client is not None else {}
+4
View File
@@ -12,6 +12,7 @@ from typing import Any
import vllm.envs as envs
from vllm import TokensPrompt
from vllm.config import VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
@@ -1282,3 +1283,6 @@ class AsyncLLM(EngineClient):
async def get_weight_version(self) -> str:
"""Return the latest committed weight version."""
return await self.engine_core.get_weight_version_async()
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
return self.engine_core.get_kv_event_sources()
+16
View File
@@ -23,6 +23,7 @@ import zmq.asyncio
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -86,6 +87,10 @@ class EngineCoreClient(ABC):
* AsyncMPClient: ZMQ + background proc EngineCore w/ asyncio (for AsyncLLM)
"""
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
"""KV-event publisher config of each ready engine, keyed by DP rank."""
return {}
@staticmethod
def make_client(
multiprocess_mode: bool,
@@ -572,6 +577,7 @@ class MPClient(EngineCoreClient):
self.vllm_config = vllm_config
self._renderer: BaseRenderer | None = renderer
self._effective_attention_block_sizes: set[int | None] = set()
self._kv_event_sources: dict[int, KVEventsConfig] = {}
# ZMQ setup.
sync_ctx = zmq.Context(io_threads=2)
@@ -860,6 +866,14 @@ class MPClient(EngineCoreClient):
else:
assert response.dp_stats_address == self.stats_update_address
if response.kv_events_config is not None:
self._kv_event_sources[response.data_parallel_rank] = (
response.kv_events_config
)
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
return dict(self._kv_event_sources)
def _process_utility_output(
output: UtilityOutput, utility_results: dict[int, AnyFuture]
@@ -1944,6 +1958,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
self.lb_engines = self.lb_engines[:new_data_parallel_size]
# Pending coordinator snapshots must use the surviving ranks too.
self.engine_ranks_managed = self.engine_ranks_managed[:new_data_parallel_size]
for rank in range(new_data_parallel_size, cur_data_parallel_size):
self._kv_event_sources.pop(rank, None)
removed_dp_size = cur_data_parallel_size - new_data_parallel_size
pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size
pause_futures = [