[Core][Frontend] Bind KV-event publishers at port 0 and expose the bound endpoints (#55844)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: mukeshbhandarkar <shell00035@gmail.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
mukeshbhandarkar
aoshen02
Claude Fable 5.1
aoshen02
parent
729ebac498
commit
90531daf6d
@@ -91,8 +91,11 @@ class MockSubscriber:
|
||||
if isinstance(replay_endpoints, str):
|
||||
replay_endpoints = [replay_endpoints]
|
||||
|
||||
ipv6 = any("[" in ep for ep in pub_endpoints + (replay_endpoints or []))
|
||||
|
||||
# Set up subscriber socket - connect to all endpoints
|
||||
self.sub = self.ctx.socket(zmq.SUB)
|
||||
self.sub.setsockopt(zmq.IPV6, ipv6)
|
||||
self.sub.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8"))
|
||||
for endpoint in pub_endpoints:
|
||||
self.sub.connect(endpoint)
|
||||
@@ -103,6 +106,7 @@ class MockSubscriber:
|
||||
if replay_endpoints:
|
||||
for replay_endpoint in replay_endpoints:
|
||||
replay = self.ctx.socket(zmq.DEALER)
|
||||
replay.setsockopt(zmq.IPV6, ipv6)
|
||||
replay.connect(replay_endpoint)
|
||||
self.replay_sockets.append(replay)
|
||||
|
||||
|
||||
@@ -332,3 +332,77 @@ def test_event_publisher_factory(random_port):
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "expected"),
|
||||
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
|
||||
)
|
||||
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
|
||||
"""Port 0 is not offset: every rank asks the OS for its own port."""
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
|
||||
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
|
||||
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
|
||||
node's IP and report endpoints a subscriber can use for events and replay."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
from vllm.utils.network_utils import split_zmq_path
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://*:0",
|
||||
replay_endpoint="tcp://*:0",
|
||||
topic="kv",
|
||||
)
|
||||
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
|
||||
try:
|
||||
resolved = [pub.get_publisher_config() for pub in pubs]
|
||||
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
|
||||
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
|
||||
assert {h for h, _ in hosts_ports} == {node_ip}
|
||||
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
|
||||
assert config.endpoint == "tcp://*:0"
|
||||
|
||||
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
|
||||
try:
|
||||
# PUB/SUB drops messages sent before the subscription lands, so
|
||||
# keep publishing until both ranks are heard.
|
||||
ranks: set[int] = set()
|
||||
deadline = time.time() + 10
|
||||
while ranks != {0, 1} and time.time() < deadline:
|
||||
for pub in pubs:
|
||||
pub.publish(create_test_events(1))
|
||||
while (result := sub.receive_one(timeout=100)) is not None:
|
||||
ranks.add(result[1].data_parallel_rank)
|
||||
assert ranks == {0, 1}
|
||||
sub.request_replay(0, socket_idx=1)
|
||||
assert sub.receive_replay(socket_idx=1)
|
||||
finally:
|
||||
sub.close()
|
||||
finally:
|
||||
for pub in pubs:
|
||||
pub.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
|
||||
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
|
||||
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
|
||||
publisher refuses instead of reporting a wildcard."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
|
||||
)
|
||||
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
|
||||
EventPublisherFactory.create(config, DP_RANK)
|
||||
|
||||
@@ -214,6 +214,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
|
||||
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
|
||||
client.eng_start_index = 0
|
||||
client._kv_event_sources = {}
|
||||
return client
|
||||
|
||||
|
||||
@@ -1490,3 +1491,57 @@ def test_ready_response_queries_executor_with_weight_transfer():
|
||||
|
||||
executor.supports_draft_weight_updates.assert_called_once()
|
||||
assert response.supports_draft_weight_updates is True
|
||||
|
||||
|
||||
def test_apply_ready_response_retains_kv_event_sources():
|
||||
"""Each engine's resolved publisher config is kept by DP rank; engines
|
||||
without KV events contribute nothing."""
|
||||
import msgspec
|
||||
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client._kv_event_sources = {}
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
return msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
max_model_len=8192,
|
||||
num_gpu_blocks=100,
|
||||
block_size=16,
|
||||
dp_stats_address=None,
|
||||
dtype="bfloat16",
|
||||
vllm_version="test",
|
||||
world_size=2,
|
||||
data_parallel_size=2,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
data_parallel_rank=rank,
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=8192,
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
kv_events_config=config,
|
||||
)
|
||||
)
|
||||
|
||||
rank1 = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:41233"
|
||||
)
|
||||
client._apply_ready_response(ready(1, rank1))
|
||||
client._apply_ready_response(ready(0, None))
|
||||
assert client.get_kv_event_sources() == {1: rank1}
|
||||
|
||||
# A rank that comes back (elastic re-add) reports its new port.
|
||||
rank1_again = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:47001"
|
||||
)
|
||||
client._apply_ready_response(ready(1, rank1_again))
|
||||
assert client.get_kv_event_sources() == {1: rank1_again}
|
||||
|
||||
Reference in New Issue
Block a user