From 90531daf6dde1c961d326d43c493477feca510c7 Mon Sep 17 00:00:00 2001 From: hangangqiang <36526001+touch869@users.noreply.github.com> Date: Sat, 19 Sep 2026 09:11:05 +0800 Subject: [PATCH] [Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844) Signed-off-by: aoshen02 Co-authored-by: mukeshbhandarkar Co-authored-by: aoshen02 Co-authored-by: Claude Fable 5.1 Co-authored-by: aoshen02 --- rust/src/server/src/routes.rs | 2 + .../src/server/src/routes/kv_event_sources.rs | 25 +++++++ rust/src/server/src/routes/tests.rs | 50 +++++++++++++ tests/distributed/conftest.py | 4 + tests/distributed/test_events.py | 74 +++++++++++++++++++ tests/v1/engine/test_engine_core_client.py | 55 ++++++++++++++ vllm/distributed/kv_events.py | 54 ++++++++++---- vllm/engine/protocol.py | 5 ++ .../entrypoints/serve/instrumentator/basic.py | 7 ++ vllm/v1/engine/async_llm.py | 4 + vllm/v1/engine/core_client.py | 16 ++++ 11 files changed, 282 insertions(+), 14 deletions(-) create mode 100644 rust/src/server/src/routes/kv_event_sources.rs diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 1cd6af3d4f..4611d0c6d6 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -6,6 +6,7 @@ mod cache; mod collective_rpc; mod health; mod inference; +mod kv_event_sources; mod load; mod lora; mod metrics; @@ -91,6 +92,7 @@ fn build_router_with_options( .route("/metrics", get(metrics::scrape)) .route("/load", get(load::load)) .route("/version", get(version::version)) + .route("/kv_event_sources", get(kv_event_sources::kv_event_sources)) // OpenAI-compatible endpoints .route("/v1/models", get(openai::list_models)) .route("/v1/completions", post(openai::completions)) diff --git a/rust/src/server/src/routes/kv_event_sources.rs b/rust/src/server/src/routes/kv_event_sources.rs new file mode 100644 index 0000000000..c3d36493a9 --- /dev/null +++ b/rust/src/server/src/routes/kv_event_sources.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::BTreeMap; +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use vllm_engine_core_client::protocol::handshake::KvEventsConfig; + +use crate::state::AppState; + +/// KV-event publisher config of each data-parallel rank, keyed by rank. +pub async fn kv_event_sources( + State(state): State>, +) -> Json> { + Json( + state + .engine_core_client() + .ready_responses() + .into_iter() + .filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?))) + .collect(), + ) +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 76d9ab8627..89ce35ce68 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -2054,6 +2054,56 @@ async fn version_returns_engine_vllm_version() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn kv_event_sources_reports_each_ready_engine() { + let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { + data_parallel_rank: 1, + kv_events_config: Some( + vllm_engine_core_client::protocol::handshake::KvEventsConfig { + enable_kv_cache_events: true, + publisher: "zmq".to_string(), + endpoint: "tcp://10.0.0.2:41233".to_string(), + replay_endpoint: None, + buffer_steps: 10_000, + hwm: 100_000, + max_queue_size: 100_000, + topic: "kv".to_string(), + }, + ), + ..default_ready_response() + }; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + let response = app + .call( + Request::builder() + .uri("/kv_event_sources") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!( + json, + json!({ + "1": { + "enable_kv_cache_events": true, + "publisher": "zmq", + "endpoint": "tcp://10.0.0.2:41233", + "replay_endpoint": null, + "buffer_steps": 10000, + "hwm": 100000, + "max_queue_size": 100000, + "topic": "kv", + } + }) + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn server_info_endpoint_is_dev_mode_only() { diff --git a/tests/distributed/conftest.py b/tests/distributed/conftest.py index 85a9d73aa5..1837d3706e 100644 --- a/tests/distributed/conftest.py +++ b/tests/distributed/conftest.py @@ -91,8 +91,11 @@ class MockSubscriber: if isinstance(replay_endpoints, str): replay_endpoints = [replay_endpoints] + ipv6 = any("[" in ep for ep in pub_endpoints + (replay_endpoints or [])) + # Set up subscriber socket - connect to all endpoints self.sub = self.ctx.socket(zmq.SUB) + self.sub.setsockopt(zmq.IPV6, ipv6) self.sub.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8")) for endpoint in pub_endpoints: self.sub.connect(endpoint) @@ -103,6 +106,7 @@ class MockSubscriber: if replay_endpoints: for replay_endpoint in replay_endpoints: replay = self.ctx.socket(zmq.DEALER) + replay.setsockopt(zmq.IPV6, ipv6) replay.connect(replay_endpoint) self.replay_sockets.append(replay) diff --git a/tests/distributed/test_events.py b/tests/distributed/test_events.py index aae80d9cce..dee0b99bb0 100644 --- a/tests/distributed/test_events.py +++ b/tests/distributed/test_events.py @@ -332,3 +332,77 @@ def test_event_publisher_factory(random_port): publisher = EventPublisherFactory.create(config, DP_RANK) assert isinstance(publisher, ZmqEventPublisher) publisher.shutdown() + + +@pytest.mark.parametrize( + ("endpoint", "expected"), + [("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")], +) +def test_offset_endpoint_port_keeps_port_zero(endpoint, expected): + """Port 0 is not offset: every rank asks the OS for its own port.""" + from vllm.distributed.kv_events import ZmqEventPublisher + + assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected + + +@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"]) +def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip): + """Two ranks binding tcp://*:0 get distinct OS-assigned ports on this + node's IP and report endpoints a subscriber can use for events and replay.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + from vllm.utils.network_utils import split_zmq_path + + from .conftest import MockSubscriber + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint="tcp://*:0", + replay_endpoint="tcp://*:0", + topic="kv", + ) + pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)] + try: + resolved = [pub.get_publisher_config() for pub in pubs] + endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)] + hosts_ports = {split_zmq_path(e)[1:] for e in endpoints} + assert {h for h, _ in hosts_ports} == {node_ip} + assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports} + assert config.endpoint == "tcp://*:0" + + sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv") + try: + # PUB/SUB drops messages sent before the subscription lands, so + # keep publishing until both ranks are heard. + ranks: set[int] = set() + deadline = time.time() + 10 + while ranks != {0, 1} and time.time() < deadline: + for pub in pubs: + pub.publish(create_test_events(1)) + while (result := sub.receive_one(timeout=100)) is not None: + ranks.add(result[1].data_parallel_rank) + assert ranks == {0, 1} + sub.request_replay(0, socket_idx=1) + assert sub.receive_replay(socket_idx=1) + finally: + sub.close() + finally: + for pub in pubs: + pub.shutdown() + + +@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"]) +def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip): + """tcp://*:0 is advertised as this node's IP; with no usable IP the + publisher refuses instead of reporting a wildcard.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0" + ) + with pytest.raises(ValueError, match="VLLM_HOST_IP"): + EventPublisherFactory.create(config, DP_RANK) diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index a153b23b10..15dec72a41 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -214,6 +214,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM client.core_engines = [bytes([i, 0]) for i in range(num_engines)] client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)] client.eng_start_index = 0 + client._kv_event_sources = {} return client @@ -1490,3 +1491,57 @@ def test_ready_response_queries_executor_with_weight_transfer(): executor.supports_draft_weight_updates.assert_called_once() assert response.supports_draft_weight_updates is True + + +def test_apply_ready_response_retains_kv_event_sources(): + """Each engine's resolved publisher config is kept by DP rank; engines + without KV events contribute nothing.""" + import msgspec + + from vllm.config.kv_events import KVEventsConfig + + client = object.__new__(MPClient) + client._effective_attention_block_sizes = set() + client._kv_event_sources = {} + client.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0), + model_config=SimpleNamespace(max_model_len=8192), + ) + + def ready(rank: int, config: KVEventsConfig | None) -> bytes: + return msgspec.msgpack.encode( + EngineCoreReadyResponse( + max_model_len=8192, + num_gpu_blocks=100, + block_size=16, + dp_stats_address=None, + dtype="bfloat16", + vllm_version="test", + world_size=2, + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=1, + decode_context_parallel_size=1, + data_parallel_rank=rank, + max_num_seqs=256, + max_num_batched_tokens=8192, + instance_id="test-instance", + supports_lora=False, + max_loras=0, + kv_events_config=config, + ) + ) + + rank1 = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:41233" + ) + client._apply_ready_response(ready(1, rank1)) + client._apply_ready_response(ready(0, None)) + assert client.get_kv_event_sources() == {1: rank1} + + # A rank that comes back (elastic re-add) reports its new port. + rank1_again = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:47001" + ) + client._apply_ready_response(ready(1, rank1_again)) + assert client.get_kv_event_sources() == {1: rank1_again} diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index 3aa2baa171..9d7ec2f827 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -17,6 +17,12 @@ import zmq from vllm.config.kv_events import KVEventsConfig from vllm.logger import init_logger +from vllm.utils.network_utils import ( + get_ip, + get_tcp_uri, + is_valid_ipv6_address, + split_zmq_path, +) from vllm.v1.core.kv_cache_utils import ExternalBlockHash logger = init_logger(__name__) @@ -349,6 +355,8 @@ class ZmqEventPublisher(EventPublisher): replay_endpoint, self._dp_rank ) assert self._endpoint is not None + self._hwm = hwm + self._socket_setup() self._publisher_config = KVEventsConfig( enable_kv_cache_events=True, publisher="zmq", @@ -359,8 +367,6 @@ class ZmqEventPublisher(EventPublisher): max_queue_size=max_queue_size, topic=topic, ) - self._hwm = hwm - self._socket_setup() # Payload self._seq_gen = count() @@ -423,16 +429,10 @@ class ZmqEventPublisher(EventPublisher): if self._pub is None: self._pub = self._ctx.socket(zmq.PUB) self._pub.set_hwm(self._hwm) - # Heuristic: bind if wildcard / * present, else connect. - # bind stable, connect volatile convention - if self._endpoint is not None and ( - "*" in self._endpoint - or "::" in self._endpoint - or self._endpoint.startswith("ipc://") - or self._endpoint.startswith("inproc://") - ): - self._pub.bind(self._endpoint) - elif self._endpoint is not None: + assert self._endpoint is not None + if self._is_bind_endpoint(self._endpoint): + self._endpoint = self._bind(self._pub, self._endpoint) + else: self._pub.connect(self._endpoint) # Set up replay socket: use ROUTER @@ -441,7 +441,31 @@ class ZmqEventPublisher(EventPublisher): # 3) works in our non‑blocking poll loop alongside PUB if self._replay_endpoint is not None: self._replay = self._ctx.socket(zmq.ROUTER) - self._replay.bind(self._replay_endpoint) + self._replay_endpoint = self._bind(self._replay, self._replay_endpoint) + + @staticmethod + def _is_bind_endpoint(endpoint: str) -> bool: + """Bind wildcard and ipc/inproc endpoints; connect to the rest.""" + return ( + "*" in endpoint + or "::" in endpoint + or endpoint.startswith(("ipc://", "inproc://")) + ) + + @staticmethod + def _bind(sock: zmq.Socket, endpoint: str) -> str: + """Bind; tcp://*:0 binds this node's IP and returns the bound endpoint.""" + if endpoint != "tcp://*:0": + sock.bind(endpoint) + return endpoint + ip = get_ip() + if ip in ("0.0.0.0", "::"): + raise ValueError("tcp://*:0 needs a node IP; set VLLM_HOST_IP") + if is_valid_ipv6_address(ip): + sock.setsockopt(zmq.IPV6, 1) + sock.bind(get_tcp_uri(ip, 0)) + _, _, port = split_zmq_path(sock.getsockopt_string(zmq.LAST_ENDPOINT)) + return get_tcp_uri(ip, int(port)) def _publisher_thread(self) -> None: """Background thread that processes the event queue.""" @@ -514,7 +538,7 @@ class ZmqEventPublisher(EventPublisher): Returns: The endpoint with the port offset by data_parallel_rank - or suffix appended + or suffix appended; a tcp port of 0 is left for the OS. """ # Do nothing if input is None or data_parallel_rank is 0 @@ -529,6 +553,8 @@ class ZmqEventPublisher(EventPublisher): last_colon_idx = endpoint.rfind(":") base_addr = endpoint[:last_colon_idx] base_port = int(endpoint[last_colon_idx + 1 :]) + if base_port == 0: + return endpoint new_port = base_port + data_parallel_rank return f"{base_addr}:{new_port}" return endpoint diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 602e2570a2..1e6011a230 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any from vllm.config import ModelConfig, VllmConfig +from vllm.config.kv_events import KVEventsConfig from vllm.distributed.weight_transfer.base import ( WeightTransferInitRequest, WeightTransferUpdateRequest, @@ -62,6 +63,10 @@ class EngineClient(ABC): @abstractmethod def dead_error(self) -> BaseException: ... + def get_kv_event_sources(self) -> dict[int, KVEventsConfig]: + """KV-event publisher config of each engine, keyed by DP rank.""" + return {} + def check_admission( # noqa: B027 self, n: int = 1, request_id: str | None = None ) -> None: diff --git a/vllm/entrypoints/serve/instrumentator/basic.py b/vllm/entrypoints/serve/instrumentator/basic.py index be091a1f43..64201db3f3 100644 --- a/vllm/entrypoints/serve/instrumentator/basic.py +++ b/vllm/entrypoints/serve/instrumentator/basic.py @@ -54,3 +54,10 @@ async def get_server_load_metrics(request: Request): async def show_version(): ver = {"version": VLLM_VERSION} return JSONResponse(content=ver) + + +@router.get("/kv_event_sources") +async def kv_event_sources(raw_request: Request): + """KV-event publisher config per DP rank; empty on render-only servers.""" + client: EngineClient | None = raw_request.app.state.engine_client + return client.get_kv_event_sources() if client is not None else {} diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 7a57d0ff18..67b8ac560d 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -12,6 +12,7 @@ from typing import Any import vllm.envs as envs from vllm import TokensPrompt from vllm.config import VllmConfig +from vllm.config.kv_events import KVEventsConfig from vllm.distributed.weight_transfer.base import ( WeightTransferInitRequest, WeightTransferUpdateRequest, @@ -1282,3 +1283,6 @@ class AsyncLLM(EngineClient): async def get_weight_version(self) -> str: """Return the latest committed weight version.""" return await self.engine_core.get_weight_version_async() + + def get_kv_event_sources(self) -> dict[int, KVEventsConfig]: + return self.engine_core.get_kv_event_sources() diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 399af7c0b9..6ad05e1f8c 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -23,6 +23,7 @@ import zmq.asyncio from vllm import envs from vllm.config import VllmConfig +from vllm.config.kv_events import KVEventsConfig from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S from vllm.logger import init_logger from vllm.lora.request import LoRARequest @@ -86,6 +87,10 @@ class EngineCoreClient(ABC): * AsyncMPClient: ZMQ + background proc EngineCore w/ asyncio (for AsyncLLM) """ + def get_kv_event_sources(self) -> dict[int, KVEventsConfig]: + """KV-event publisher config of each ready engine, keyed by DP rank.""" + return {} + @staticmethod def make_client( multiprocess_mode: bool, @@ -572,6 +577,7 @@ class MPClient(EngineCoreClient): self.vllm_config = vllm_config self._renderer: BaseRenderer | None = renderer self._effective_attention_block_sizes: set[int | None] = set() + self._kv_event_sources: dict[int, KVEventsConfig] = {} # ZMQ setup. sync_ctx = zmq.Context(io_threads=2) @@ -860,6 +866,14 @@ class MPClient(EngineCoreClient): else: assert response.dp_stats_address == self.stats_update_address + if response.kv_events_config is not None: + self._kv_event_sources[response.data_parallel_rank] = ( + response.kv_events_config + ) + + def get_kv_event_sources(self) -> dict[int, KVEventsConfig]: + return dict(self._kv_event_sources) + def _process_utility_output( output: UtilityOutput, utility_results: dict[int, AnyFuture] @@ -1944,6 +1958,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient): self.lb_engines = self.lb_engines[:new_data_parallel_size] # Pending coordinator snapshots must use the surviving ranks too. self.engine_ranks_managed = self.engine_ranks_managed[:new_data_parallel_size] + for rank in range(new_data_parallel_size, cur_data_parallel_size): + self._kv_event_sources.pop(rank, None) removed_dp_size = cur_data_parallel_size - new_data_parallel_size pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size pause_futures = [