[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
False,
),
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
],
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
) -> None:
"""The DP master node holds a coordination store for both engine backends,
so engines there pick world-group ports at bind time. Every other
deployment keeps the pre-allocated ports and must not hold one."""
so engines there pick world-group ports at bind time. Elastic EP also
creates its coordination store before startup. The remaining cases
below keep the pre-allocated ports and must not hold a store."""
class FakeManager:
def __init__(self, **kwargs) -> None:
+15 -1
View File
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
def setsockopt(self, *_args, **_kwargs):
pass
def getsockopt_string(self, _option):
return "inproc://dummy"
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
monkeypatch.setattr(
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
@@ -448,7 +451,7 @@ def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
@@ -456,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
client.coordinator_input_address = None
client.coordinator_output_address = None
payload = msgspec.msgpack.encode(
EngineCoreReadyResponse(
@@ -478,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
instance_id="test-instance",
supports_lora=False,
max_loras=0,
coord_store_port=1234,
coordinator_input_address="tcp://127.0.0.1:1235",
coordinator_output_address="tcp://127.0.0.1:1236",
)
)
fields = msgspec.msgpack.decode(payload)
@@ -498,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
assert client.vllm_config.parallel_config._coord_store_port == 1234
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
def test_apply_ready_response_syncs_mamba_block_size():
@@ -508,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
@@ -1578,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
@@ -281,6 +281,7 @@ class _FinishedProcess:
def test_wait_for_engine_startup_reports_watched_process_exit():
"""Update the startup test argument so it reaches frontend failure handling."""
ctx = zmq.Context()
handshake_socket = ctx.socket(zmq.ROUTER)
recv, send = connection.Pipe(duplex=False)