[Feature][EPD] Support dynamic EPD (EC-connector) proxy (#54176)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-20 12:24:41 +08:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex
parent 1596fa5f88
commit 70fc7bd695
8 changed files with 1168 additions and 72 deletions
@@ -0,0 +1,279 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing and registration behaviour of the EPD proxy."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, Mock
import msgspec
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from examples.disaggregated.disaggregated_encoder import (
disagg_epd_proxy as proxy_module,
)
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
EPDProxy,
EPDProxyConfig,
InstanceRecord,
InstanceRegistry,
InstanceRole,
build_app,
extract_mm_items,
)
ENCODE = InstanceRole.ENCODE
PREFILL = InstanceRole.PREFILL
DECODE = InstanceRole.DECODE
PD = InstanceRole.PREFILL_DECODE
IMAGE_ITEM = {"type": "image_url", "image_url": {"url": "http://img/0.png"}}
@pytest.fixture
def proxy():
registry = InstanceRegistry(probe_interval=0)
return EPDProxy(registry)
class TestRouting:
def test_no_decode_instance_is_service_unavailable(self, proxy):
"""The proxy comes up before anything registers, so this is normal."""
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
def test_media_without_an_encoder_is_service_unavailable(self, proxy):
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
assert proxy.route(num_items=0).decode.url == "http://d0:8000"
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=1)
assert excinfo.value.status_code == 503
def test_prefill_is_optional(self, proxy):
"""An E+PD deployment registers no prefill instance at all."""
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
assert proxy.route(num_items=0).prefill is None
def test_encoder_roster_tracks_new_instances(self, proxy):
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
for index in range(2):
proxy.registry.register(InstanceRecord(ENCODE, f"http://e{index}:8000"))
assert proxy.route(num_items=3).encoder_urls == [
"http://e0:8000",
"http://e1:8000",
]
proxy.registry.unregister("http://e0:8000")
assert proxy.route(num_items=3).encoder_urls == ["http://e1:8000"]
@pytest.mark.parametrize("unregister", [False, True])
def test_standalone_decode_always_requires_healthy_prefill(self, unregister):
app = build_app()
proxy = app.state.proxy
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
proxy.registry.register(InstanceRecord(PREFILL, "http://p0:8000"))
route = proxy.route(num_items=0)
assert route.decode.url == "http://d0:8000"
assert route.consumer.url == "http://p0:8000"
if unregister:
proxy.registry.unregister("http://p0:8000")
else:
for _ in range(proxy.registry._fail_threshold):
proxy.registry._on_probe_failure(route.prefill, 0)
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
class TestConsumerAddress:
"""Which stage receives the embedding depends on the topology."""
def test_shared_storage_connectors_name_no_target(self, proxy):
"""Rotate replicas without requiring a push endpoint."""
proxy.registry.register(InstanceRecord(PD, "http://d0:8000", dp_size=2))
route = proxy.route(num_items=0)
assert route.consumer_zmq is None
assert route.dp_rank == 0
assert proxy.route(num_items=0).dp_rank == 1
def test_decode_is_the_consumer_when_prefill_is_not_split_out(self, proxy):
proxy.registry.register(
InstanceRecord(PD, "http://d0:8000", ec_zmq_addrs=["tcp://d0:20001"])
)
assert proxy.route(num_items=0).consumer_zmq == "tcp://d0:20001"
def test_prefill_is_the_consumer_when_it_is_split_out(self, proxy):
"""In E+P+D the prefill instance consumes the embedding, not decode.
Reading the address out of whichever list it came from -- rather than
from the record that reported one -- sends the push to the wrong
instance in this topology.
"""
proxy.registry.register(
InstanceRecord(PREFILL, "http://p0:8000", ec_zmq_addrs=["tcp://p0:20001"])
)
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
assert proxy.route(num_items=0).consumer_zmq == "tcp://p0:20001"
class TestProxyApi:
@pytest.fixture
def client(self, monkeypatch):
monkeypatch.setenv("ADMIN_API_KEY", "test-key")
config = EPDProxyConfig(probe_interval=0)
with TestClient(build_app(config)) as client:
client.headers["X-API-Key"] = "test-key"
yield client
def test_proxy_serves_before_anything_registers(self, client):
assert client.get("/health").status_code == 200
assert client.get("/instances").json()["encode"]["live"] == []
def test_requests_are_refused_until_a_decoder_registers(self, client):
response = client.post(
"/v1/chat/completions", json={"model": "m", "messages": []}
)
assert response.status_code == 503
assert client.get("/v1/models").status_code == 503
def test_http_registration_keeps_consumer_replica_and_address_together(
self, client
):
payload: dict[str, Any] = {
"role": "prefill_decode",
"url": "http://d0:8000/",
"dp_size": 2,
"ec_zmq_addrs": ["tcp://d0:20001", "tcp://d0:20003"],
}
assert client.post("/instances", json=payload).json() == {"registered": True}
assert client.post("/instances", json=payload).json() == {"registered": False}
proxy = client.app.state.proxy
for rank in [0, 1, 0]:
route = proxy.route(0)
assert route.dp_rank == rank
assert route.consumer_zmq == payload["ec_zmq_addrs"][rank]
assert client.delete("/instances", params={"url": payload["url"]}).json() == {
"removed": True
}
assert proxy.registry.pick(PD) is None
def test_registration_rejects_invalid_topology(self, client):
cases: list[tuple[dict[str, Any], int]] = [
({"dp_size": 0}, 422),
({"dp_size": 2, "ec_zmq_addrs": ["tcp://d0:20001"]}, 422),
({"role": "decode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
({"role": "encode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
]
for fields, status in cases:
response = client.post(
"/instances",
json={"url": "http://d0:8000", "role": "prefill_decode", **fields},
)
assert response.status_code == status
def test_registration_requires_admin_key(self, client):
client.headers.pop("X-API-Key")
assert (
client.post(
"/instances", json={"role": "encode", "url": "http://e0:8000"}
).status_code
== 403
)
assert (
client.delete("/instances", params={"url": "http://e0:8000"}).status_code
== 403
)
@pytest.mark.parametrize("split_role", ["prefill", "decode"])
@pytest.mark.parametrize("combined_first", [False, True])
def test_registration_rejects_mixed_topologies(
self, client, split_role, combined_first
):
roles = ["prefill_decode", split_role]
if not combined_first:
roles.reverse()
first = {"role": roles[0], "url": "http://first:8000"}
second = {"role": roles[1], "url": "http://second:8000"}
assert client.post("/instances", json=first).status_code == 200
assert client.post("/instances", json=second).status_code == 409
# Health eviction must not silently permit a topology switch.
registry = client.app.state.registry
record = registry._live[first["url"]]
for _ in range(registry._fail_threshold):
registry._on_probe_failure(record, 0)
assert client.post("/instances", json=second).status_code == 409
assert client.delete("/instances", params={"url": first["url"]}).json() == {
"removed": True
}
assert client.post("/instances", json=second).status_code == 200
def test_extract_mm_items_finds_media_across_messages():
req = {
"messages": [
{"role": "user", "content": "plain text"},
{"role": "user", "content": [IMAGE_ITEM, {"type": "text", "text": "hi"}]},
{"role": "user", "content": [IMAGE_ITEM]},
]
}
assert extract_mm_items(req) == [IMAGE_ITEM, IMAGE_ITEM]
@pytest.mark.parametrize("push", [False, True])
def test_encoder_handles_and_json_metadata_survive_rewrite(proxy, push, monkeypatch):
"""Keep main's hash-keyed handles and per-item push IDs after the proxy move."""
proxy.registry.register(InstanceRecord(ENCODE, "http://encoder"))
proxy.registry.register(
InstanceRecord(
PD,
"http://decode",
ec_zmq_addrs=["tcp://decode:14579"] if push else [],
)
)
handle = {
"metadata": {"image_grid_thw": [[1, 2, 3]]},
"item_indices": [0, 1],
"peer_port": 5601,
}
response = Mock(
status=200,
read=AsyncMock(
return_value=msgspec.json.encode(
{"ec_transfer_params": {"engine-hash": handle}}
)
),
)
session = Mock(post=AsyncMock(return_value=response))
monkeypatch.setattr(proxy_module, "encode_session", session)
original = {"messages": [{"role": "user", "content": [IMAGE_ITEM, IMAGE_ITEM]}]}
route = proxy.route(2)
prepared, _, _ = asyncio.run(
proxy_module.prepare_for_decode(
original, "request", route.encoder_urls, None, route.consumer_zmq
)
)
assert "ec_transfer_params" not in original
assert prepared["ec_transfer_params"]["engine-hash"] == handle
items = prepared["ec_transfer_params"]["ec_items"]
assert [item["mm_hash"] for item in items] == ["engine-hash", "engine-hash"]
assert items[0]["transfer_id"] != items[1]["transfer_id"]
assert session.post.call_count == 1
sent = msgspec.json.decode(session.post.call_args.kwargs["data"]).get(
"ec_transfer_params"
)
if push:
assert sent == {
"consumer_zmq": "tcp://decode:14579",
"ec_items": [{"transfer_id": item["transfer_id"]} for item in items],
}
else:
assert sent is None
for index in range(2):
assert prepared["messages"][0]["content"][index]["image_embeds"] == {
"image_grid_thw": [1, 2, 3]
}
@@ -8,6 +8,7 @@ change to them is what these tests catch.
import asyncio
import importlib.util
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -27,6 +28,7 @@ def proxy():
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@@ -48,7 +50,7 @@ def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
"""
served = [{"remote_block_ids": [1, 2]}, {}]
async def _stage(req_data, p_url, req_id):
async def _stage(req_data, p_url, req_id, dp_rank=None):
assert "kv_transfer_params" not in req_data
return _Response(served.pop(0))
@@ -222,7 +224,7 @@ def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch
_EncoderSession([{"encoder-side-hash": handle}, {}]),
)
async def _no_prefill(req_data, p_url, req_id):
async def _no_prefill(req_data, p_url, req_id, dp_rank=None):
return req_data
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
@@ -280,6 +282,7 @@ def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("prefill", [False, True])
@pytest.mark.parametrize("dynamic", [False, True])
@pytest.mark.parametrize(
"request_options",
[
@@ -293,7 +296,7 @@ def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
],
)
async def test_http_roundtrip_preserves_payload_and_response_bytes(
proxy, monkeypatch, stream, prefill, request_options
proxy, monkeypatch, stream, prefill, dynamic, request_options
):
"""Preserve media and scheduling context across HTTP hops and retries."""
seen: dict[str, list[dict]] = {"encode": [], "prefill": [], "decode": []}
@@ -307,6 +310,10 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
async def backend(request):
stage = request.match_info["stage"]
if dynamic:
consumer = "prefill" if prefill else "decode"
expected_rank = "0" if stage == consumer else None
assert request.headers.get("X-data-parallel-rank") == expected_rank
assert request.content_type == "application/json"
body = await request.json()
seen[stage].append(body)
@@ -354,6 +361,10 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
monkeypatch.setattr(proxy.app.state, "ec_consumer_dp_size", 1, raising=False)
monkeypatch.setattr(proxy, "DECODE_RETRIES", 1)
monkeypatch.setattr(proxy, "NO_REWRITE", False)
target_app = proxy.app
if dynamic:
monkeypatch.setenv("ADMIN_API_KEY", "test-key")
target_app = proxy.build_app(proxy.EPDProxyConfig(probe_interval=0))
item = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
body = {
"model": "test",
@@ -377,9 +388,37 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
await proxy.on_startup()
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=proxy.app), base_url="http://proxy"
transport=httpx.ASGITransport(app=target_app), base_url="http://proxy"
) as client:
if dynamic:
registrations = [("encode", "encode")]
registrations += (
[("prefill", "prefill"), ("decode", "decode")]
if prefill
else [("prefill_decode", "decode")]
)
for role, stage in registrations:
consumer = "prefill" if prefill else "decode"
registered = await client.post(
"/instances",
headers={"X-API-Key": "test-key"},
json={
"role": role,
"url": str(server.make_url(f"/{stage}")),
"dp_size": 2 if stage == consumer else 1,
},
)
assert registered.status_code == 200
response = await client.post("/v1/chat/completions", json=body)
if dynamic:
removed = await client.delete(
"/instances",
headers={"X-API-Key": "test-key"},
params={"url": str(server.make_url("/decode"))},
)
assert removed.json() == {"removed": True}
unavailable = await client.post("/v1/chat/completions", json=body)
assert unavailable.status_code == 503
assert response.status_code == 200
assert response.content == expected
assert response.headers["content-type"].startswith(
@@ -14,6 +14,7 @@ across calls so consecutive requests keep advancing through every URL.
"""
import importlib.util
import sys
from collections import Counter
from pathlib import Path
@@ -27,6 +28,7 @@ def _load_proxy_module():
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_under_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Dynamic registration and liveness for the EPD proxy's instance registry."""
from unittest.mock import patch
import pytest
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
InstanceRecord,
InstanceRegistry,
InstanceRole,
)
ENCODE = InstanceRole.ENCODE
DECODE = InstanceRole.DECODE
@pytest.fixture
def registry():
return InstanceRegistry(probe_interval=0, fail_threshold=3, evicted_ttl=60.0)
def _fill(registry, role, count, prefix="http://h"):
for index in range(count):
registry.register(InstanceRecord(role, f"{prefix}{index}:8000"))
async def _probe_round(registry, healthy: set[str], now: float = 0.0):
"""Run one probe pass where only `healthy` URLs answer."""
async def fake_probe(self, session, url):
return url in healthy
with (
patch.object(InstanceRegistry, "_probe", fake_probe),
patch(
"examples.disaggregated.disaggregated_encoder.disagg_epd_proxy.time.monotonic",
return_value=now,
),
):
await registry._probe_once(session=None)
class TestRegistration:
def test_registering_is_idempotent_and_reports_novelty(self, registry):
record = InstanceRecord(ENCODE, "http://e0:8000")
assert registry.register(record) is True
assert registry.register(record) is False
assert registry.urls(ENCODE) == ["http://e0:8000"]
def test_transfer_addresses_stay_with_the_registered_consumer(self, registry):
"""Roster updates must not shift a consumer's transfer addresses."""
registry.register(InstanceRecord(DECODE, "http://d0:8000"))
registry.register(
InstanceRecord(
DECODE,
"http://d0:8000",
ec_zmq_addrs=["tcp://d0:20001", "tcp://d0:20002"],
dp_size=2,
)
)
picked = registry.pick(DECODE)
assert picked.ec_zmq_addrs == ["tcp://d0:20001", "tcp://d0:20002"]
assert picked.dp_size == 2
def test_unregister_is_terminal(self, registry):
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
assert registry.unregister("http://e0:8000") is True
assert registry.unregister("http://e0:8000") is False
assert registry.pick(ENCODE) is None
def test_round_robin_survives_registration(self, registry):
_fill(registry, ENCODE, 2)
assert [record.url for record in registry.pick_many(ENCODE, 2)] == [
"http://h0:8000",
"http://h1:8000",
]
registry.register(InstanceRecord(ENCODE, "http://h2:8000"))
assert registry.pick(ENCODE).url == "http://h2:8000"
class TestLiveness:
@pytest.mark.asyncio
async def test_identical_reregistration_preserves_inflight_probe_results(
self, registry
):
"""Repeated registrations must not suppress eviction or recovery."""
url = "http://e0:8000"
registry.register(InstanceRecord(ENCODE, url))
healthy = False
async def probe(self, session, url):
registry.register(InstanceRecord(ENCODE, url))
return healthy
with patch.object(InstanceRegistry, "_probe", probe):
for _ in range(registry._fail_threshold):
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [], "evicted": [url]}
healthy = True
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [url], "evicted": []}
@pytest.mark.asyncio
async def test_consecutive_failures_stop_routing(self, registry):
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
registry.register(InstanceRecord(ENCODE, "http://e1:8000"))
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy={"http://e1:8000"})
assert registry.urls(ENCODE) == ["http://e1:8000"]
assert registry.status()["encode"]["evicted"] == ["http://e0:8000"]
@pytest.mark.asyncio
@pytest.mark.parametrize("role", [ENCODE, DECODE])
async def test_a_recovered_instance_rejoins_without_re_registering(
self, registry, role
):
"""Restarting every encoder to recover from a blip is not acceptable."""
registry.register(InstanceRecord(role, "http://e0:8000"))
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
assert registry.urls(role) == []
await _probe_round(registry, healthy={"http://e0:8000"}, now=1)
assert registry.urls(role) == ["http://e0:8000"]
@pytest.mark.asyncio
async def test_re_registration_does_not_override_failed_health_checks(
self, registry
):
record = InstanceRecord(ENCODE, "http://e0:8000")
registry.register(record)
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
registry.register(record)
assert registry.urls(ENCODE) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("healthy", [False, True])
async def test_inflight_probe_cannot_restore_unregistered_instance(
self, registry, healthy
):
record = InstanceRecord(ENCODE, "http://e0:8000")
registry.register(record)
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
async def probe(self, session, url):
registry.unregister(url)
return healthy
with patch.object(InstanceRegistry, "_probe", probe):
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [], "evicted": []}