[Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: mukeshbhandarkar <shell00035@gmail.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
mukeshbhandarkar
aoshen02
Claude Fable 5.1
aoshen02
parent
729ebac498
commit
90531daf6d
@@ -6,6 +6,7 @@ mod cache;
|
||||
mod collective_rpc;
|
||||
mod health;
|
||||
mod inference;
|
||||
mod kv_event_sources;
|
||||
mod load;
|
||||
mod lora;
|
||||
mod metrics;
|
||||
@@ -91,6 +92,7 @@ fn build_router_with_options(
|
||||
.route("/metrics", get(metrics::scrape))
|
||||
.route("/load", get(load::load))
|
||||
.route("/version", get(version::version))
|
||||
.route("/kv_event_sources", get(kv_event_sources::kv_event_sources))
|
||||
// OpenAI-compatible endpoints
|
||||
.route("/v1/models", get(openai::list_models))
|
||||
.route("/v1/completions", post(openai::completions))
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use vllm_engine_core_client::protocol::handshake::KvEventsConfig;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// KV-event publisher config of each data-parallel rank, keyed by rank.
|
||||
pub async fn kv_event_sources(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<BTreeMap<u32, KvEventsConfig>> {
|
||||
Json(
|
||||
state
|
||||
.engine_core_client()
|
||||
.ready_responses()
|
||||
.into_iter()
|
||||
.filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?)))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
@@ -2054,6 +2054,56 @@ async fn version_returns_engine_vllm_version() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn kv_event_sources_reports_each_ready_engine() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
data_parallel_rank: 1,
|
||||
kv_events_config: Some(
|
||||
vllm_engine_core_client::protocol::handshake::KvEventsConfig {
|
||||
enable_kv_cache_events: true,
|
||||
publisher: "zmq".to_string(),
|
||||
endpoint: "tcp://10.0.0.2:41233".to_string(),
|
||||
replay_endpoint: None,
|
||||
buffer_steps: 10_000,
|
||||
hwm: 100_000,
|
||||
max_queue_size: 100_000,
|
||||
topic: "kv".to_string(),
|
||||
},
|
||||
),
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/kv_event_sources")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(
|
||||
json,
|
||||
json!({
|
||||
"1": {
|
||||
"enable_kv_cache_events": true,
|
||||
"publisher": "zmq",
|
||||
"endpoint": "tcp://10.0.0.2:41233",
|
||||
"replay_endpoint": null,
|
||||
"buffer_steps": 10000,
|
||||
"hwm": 100000,
|
||||
"max_queue_size": 100000,
|
||||
"topic": "kv",
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn server_info_endpoint_is_dev_mode_only() {
|
||||
|
||||
@@ -91,8 +91,11 @@ class MockSubscriber:
|
||||
if isinstance(replay_endpoints, str):
|
||||
replay_endpoints = [replay_endpoints]
|
||||
|
||||
ipv6 = any("[" in ep for ep in pub_endpoints + (replay_endpoints or []))
|
||||
|
||||
# Set up subscriber socket - connect to all endpoints
|
||||
self.sub = self.ctx.socket(zmq.SUB)
|
||||
self.sub.setsockopt(zmq.IPV6, ipv6)
|
||||
self.sub.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8"))
|
||||
for endpoint in pub_endpoints:
|
||||
self.sub.connect(endpoint)
|
||||
@@ -103,6 +106,7 @@ class MockSubscriber:
|
||||
if replay_endpoints:
|
||||
for replay_endpoint in replay_endpoints:
|
||||
replay = self.ctx.socket(zmq.DEALER)
|
||||
replay.setsockopt(zmq.IPV6, ipv6)
|
||||
replay.connect(replay_endpoint)
|
||||
self.replay_sockets.append(replay)
|
||||
|
||||
|
||||
@@ -332,3 +332,77 @@ def test_event_publisher_factory(random_port):
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "expected"),
|
||||
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
|
||||
)
|
||||
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
|
||||
"""Port 0 is not offset: every rank asks the OS for its own port."""
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
|
||||
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
|
||||
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
|
||||
node's IP and report endpoints a subscriber can use for events and replay."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
from vllm.utils.network_utils import split_zmq_path
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://*:0",
|
||||
replay_endpoint="tcp://*:0",
|
||||
topic="kv",
|
||||
)
|
||||
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
|
||||
try:
|
||||
resolved = [pub.get_publisher_config() for pub in pubs]
|
||||
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
|
||||
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
|
||||
assert {h for h, _ in hosts_ports} == {node_ip}
|
||||
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
|
||||
assert config.endpoint == "tcp://*:0"
|
||||
|
||||
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
|
||||
try:
|
||||
# PUB/SUB drops messages sent before the subscription lands, so
|
||||
# keep publishing until both ranks are heard.
|
||||
ranks: set[int] = set()
|
||||
deadline = time.time() + 10
|
||||
while ranks != {0, 1} and time.time() < deadline:
|
||||
for pub in pubs:
|
||||
pub.publish(create_test_events(1))
|
||||
while (result := sub.receive_one(timeout=100)) is not None:
|
||||
ranks.add(result[1].data_parallel_rank)
|
||||
assert ranks == {0, 1}
|
||||
sub.request_replay(0, socket_idx=1)
|
||||
assert sub.receive_replay(socket_idx=1)
|
||||
finally:
|
||||
sub.close()
|
||||
finally:
|
||||
for pub in pubs:
|
||||
pub.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
|
||||
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
|
||||
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
|
||||
publisher refuses instead of reporting a wildcard."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
|
||||
)
|
||||
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
|
||||
EventPublisherFactory.create(config, DP_RANK)
|
||||
|
||||
@@ -214,6 +214,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
|
||||
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
|
||||
client.eng_start_index = 0
|
||||
client._kv_event_sources = {}
|
||||
return client
|
||||
|
||||
|
||||
@@ -1490,3 +1491,57 @@ def test_ready_response_queries_executor_with_weight_transfer():
|
||||
|
||||
executor.supports_draft_weight_updates.assert_called_once()
|
||||
assert response.supports_draft_weight_updates is True
|
||||
|
||||
|
||||
def test_apply_ready_response_retains_kv_event_sources():
|
||||
"""Each engine's resolved publisher config is kept by DP rank; engines
|
||||
without KV events contribute nothing."""
|
||||
import msgspec
|
||||
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client._kv_event_sources = {}
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
return msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
max_model_len=8192,
|
||||
num_gpu_blocks=100,
|
||||
block_size=16,
|
||||
dp_stats_address=None,
|
||||
dtype="bfloat16",
|
||||
vllm_version="test",
|
||||
world_size=2,
|
||||
data_parallel_size=2,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
data_parallel_rank=rank,
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=8192,
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
kv_events_config=config,
|
||||
)
|
||||
)
|
||||
|
||||
rank1 = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:41233"
|
||||
)
|
||||
client._apply_ready_response(ready(1, rank1))
|
||||
client._apply_ready_response(ready(0, None))
|
||||
assert client.get_kv_event_sources() == {1: rank1}
|
||||
|
||||
# A rank that comes back (elastic re-add) reports its new port.
|
||||
rank1_again = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:47001"
|
||||
)
|
||||
client._apply_ready_response(ready(1, rank1_again))
|
||||
assert client.get_kv_event_sources() == {1: rank1_again}
|
||||
|
||||
@@ -17,6 +17,12 @@ import zmq
|
||||
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import (
|
||||
get_ip,
|
||||
get_tcp_uri,
|
||||
is_valid_ipv6_address,
|
||||
split_zmq_path,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import ExternalBlockHash
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -349,6 +355,8 @@ class ZmqEventPublisher(EventPublisher):
|
||||
replay_endpoint, self._dp_rank
|
||||
)
|
||||
assert self._endpoint is not None
|
||||
self._hwm = hwm
|
||||
self._socket_setup()
|
||||
self._publisher_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
@@ -359,8 +367,6 @@ class ZmqEventPublisher(EventPublisher):
|
||||
max_queue_size=max_queue_size,
|
||||
topic=topic,
|
||||
)
|
||||
self._hwm = hwm
|
||||
self._socket_setup()
|
||||
|
||||
# Payload
|
||||
self._seq_gen = count()
|
||||
@@ -423,16 +429,10 @@ class ZmqEventPublisher(EventPublisher):
|
||||
if self._pub is None:
|
||||
self._pub = self._ctx.socket(zmq.PUB)
|
||||
self._pub.set_hwm(self._hwm)
|
||||
# Heuristic: bind if wildcard / * present, else connect.
|
||||
# bind stable, connect volatile convention
|
||||
if self._endpoint is not None and (
|
||||
"*" in self._endpoint
|
||||
or "::" in self._endpoint
|
||||
or self._endpoint.startswith("ipc://")
|
||||
or self._endpoint.startswith("inproc://")
|
||||
):
|
||||
self._pub.bind(self._endpoint)
|
||||
elif self._endpoint is not None:
|
||||
assert self._endpoint is not None
|
||||
if self._is_bind_endpoint(self._endpoint):
|
||||
self._endpoint = self._bind(self._pub, self._endpoint)
|
||||
else:
|
||||
self._pub.connect(self._endpoint)
|
||||
|
||||
# Set up replay socket: use ROUTER
|
||||
@@ -441,7 +441,31 @@ class ZmqEventPublisher(EventPublisher):
|
||||
# 3) works in our non‑blocking poll loop alongside PUB
|
||||
if self._replay_endpoint is not None:
|
||||
self._replay = self._ctx.socket(zmq.ROUTER)
|
||||
self._replay.bind(self._replay_endpoint)
|
||||
self._replay_endpoint = self._bind(self._replay, self._replay_endpoint)
|
||||
|
||||
@staticmethod
|
||||
def _is_bind_endpoint(endpoint: str) -> bool:
|
||||
"""Bind wildcard and ipc/inproc endpoints; connect to the rest."""
|
||||
return (
|
||||
"*" in endpoint
|
||||
or "::" in endpoint
|
||||
or endpoint.startswith(("ipc://", "inproc://"))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _bind(sock: zmq.Socket, endpoint: str) -> str:
|
||||
"""Bind; tcp://*:0 binds this node's IP and returns the bound endpoint."""
|
||||
if endpoint != "tcp://*:0":
|
||||
sock.bind(endpoint)
|
||||
return endpoint
|
||||
ip = get_ip()
|
||||
if ip in ("0.0.0.0", "::"):
|
||||
raise ValueError("tcp://*:0 needs a node IP; set VLLM_HOST_IP")
|
||||
if is_valid_ipv6_address(ip):
|
||||
sock.setsockopt(zmq.IPV6, 1)
|
||||
sock.bind(get_tcp_uri(ip, 0))
|
||||
_, _, port = split_zmq_path(sock.getsockopt_string(zmq.LAST_ENDPOINT))
|
||||
return get_tcp_uri(ip, int(port))
|
||||
|
||||
def _publisher_thread(self) -> None:
|
||||
"""Background thread that processes the event queue."""
|
||||
@@ -514,7 +538,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
|
||||
Returns:
|
||||
The endpoint with the port offset by data_parallel_rank
|
||||
or suffix appended
|
||||
or suffix appended; a tcp port of 0 is left for the OS.
|
||||
|
||||
"""
|
||||
# Do nothing if input is None or data_parallel_rank is 0
|
||||
@@ -529,6 +553,8 @@ class ZmqEventPublisher(EventPublisher):
|
||||
last_colon_idx = endpoint.rfind(":")
|
||||
base_addr = endpoint[:last_colon_idx]
|
||||
base_port = int(endpoint[last_colon_idx + 1 :])
|
||||
if base_port == 0:
|
||||
return endpoint
|
||||
new_port = base_port + data_parallel_rank
|
||||
return f"{base_addr}:{new_port}"
|
||||
return endpoint
|
||||
|
||||
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
@@ -62,6 +63,10 @@ class EngineClient(ABC):
|
||||
@abstractmethod
|
||||
def dead_error(self) -> BaseException: ...
|
||||
|
||||
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
|
||||
"""KV-event publisher config of each engine, keyed by DP rank."""
|
||||
return {}
|
||||
|
||||
def check_admission( # noqa: B027
|
||||
self, n: int = 1, request_id: str | None = None
|
||||
) -> None:
|
||||
|
||||
@@ -54,3 +54,10 @@ async def get_server_load_metrics(request: Request):
|
||||
async def show_version():
|
||||
ver = {"version": VLLM_VERSION}
|
||||
return JSONResponse(content=ver)
|
||||
|
||||
|
||||
@router.get("/kv_event_sources")
|
||||
async def kv_event_sources(raw_request: Request):
|
||||
"""KV-event publisher config per DP rank; empty on render-only servers."""
|
||||
client: EngineClient | None = raw_request.app.state.engine_client
|
||||
return client.get_kv_event_sources() if client is not None else {}
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
import vllm.envs as envs
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
@@ -1282,3 +1283,6 @@ class AsyncLLM(EngineClient):
|
||||
async def get_weight_version(self) -> str:
|
||||
"""Return the latest committed weight version."""
|
||||
return await self.engine_core.get_weight_version_async()
|
||||
|
||||
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
|
||||
return self.engine_core.get_kv_event_sources()
|
||||
|
||||
@@ -23,6 +23,7 @@ import zmq.asyncio
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.envs import VLLM_ENGINE_READY_TIMEOUT_S
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -86,6 +87,10 @@ class EngineCoreClient(ABC):
|
||||
* AsyncMPClient: ZMQ + background proc EngineCore w/ asyncio (for AsyncLLM)
|
||||
"""
|
||||
|
||||
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
|
||||
"""KV-event publisher config of each ready engine, keyed by DP rank."""
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def make_client(
|
||||
multiprocess_mode: bool,
|
||||
@@ -572,6 +577,7 @@ class MPClient(EngineCoreClient):
|
||||
self.vllm_config = vllm_config
|
||||
self._renderer: BaseRenderer | None = renderer
|
||||
self._effective_attention_block_sizes: set[int | None] = set()
|
||||
self._kv_event_sources: dict[int, KVEventsConfig] = {}
|
||||
|
||||
# ZMQ setup.
|
||||
sync_ctx = zmq.Context(io_threads=2)
|
||||
@@ -860,6 +866,14 @@ class MPClient(EngineCoreClient):
|
||||
else:
|
||||
assert response.dp_stats_address == self.stats_update_address
|
||||
|
||||
if response.kv_events_config is not None:
|
||||
self._kv_event_sources[response.data_parallel_rank] = (
|
||||
response.kv_events_config
|
||||
)
|
||||
|
||||
def get_kv_event_sources(self) -> dict[int, KVEventsConfig]:
|
||||
return dict(self._kv_event_sources)
|
||||
|
||||
|
||||
def _process_utility_output(
|
||||
output: UtilityOutput, utility_results: dict[int, AnyFuture]
|
||||
@@ -1944,6 +1958,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
self.lb_engines = self.lb_engines[:new_data_parallel_size]
|
||||
# Pending coordinator snapshots must use the surviving ranks too.
|
||||
self.engine_ranks_managed = self.engine_ranks_managed[:new_data_parallel_size]
|
||||
for rank in range(new_data_parallel_size, cur_data_parallel_size):
|
||||
self._kv_event_sources.pop(rank, None)
|
||||
removed_dp_size = cur_data_parallel_size - new_data_parallel_size
|
||||
pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size
|
||||
pause_futures = [
|
||||
|
||||
Reference in New Issue
Block a user