[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
|
||||
False,
|
||||
),
|
||||
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
|
||||
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
|
||||
],
|
||||
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
|
||||
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
|
||||
) -> None:
|
||||
"""The DP master node holds a coordination store for both engine backends,
|
||||
so engines there pick world-group ports at bind time. Every other
|
||||
deployment keeps the pre-allocated ports and must not hold one."""
|
||||
so engines there pick world-group ports at bind time. Elastic EP also
|
||||
creates its coordination store before startup. The remaining cases
|
||||
below keep the pre-allocated ports and must not hold a store."""
|
||||
|
||||
class FakeManager:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
|
||||
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def getsockopt_string(self, _option):
|
||||
return "inproc://dummy"
|
||||
|
||||
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
|
||||
monkeypatch.setattr(
|
||||
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
|
||||
@@ -448,7 +451,7 @@ def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
@@ -456,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
client.coordinator_input_address = None
|
||||
client.coordinator_output_address = None
|
||||
|
||||
payload = msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
@@ -478,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
coord_store_port=1234,
|
||||
coordinator_input_address="tcp://127.0.0.1:1235",
|
||||
coordinator_output_address="tcp://127.0.0.1:1236",
|
||||
)
|
||||
)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
@@ -498,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
assert client.vllm_config.parallel_config._coord_store_port == 1234
|
||||
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
|
||||
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
@@ -508,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
|
||||
@@ -1578,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
|
||||
@@ -281,6 +281,7 @@ class _FinishedProcess:
|
||||
|
||||
|
||||
def test_wait_for_engine_startup_reports_watched_process_exit():
|
||||
"""Update the startup test argument so it reaches frontend failure handling."""
|
||||
ctx = zmq.Context()
|
||||
handshake_socket = ctx.socket(zmq.ROUTER)
|
||||
recv, send = connection.Pipe(duplex=False)
|
||||
|
||||
Reference in New Issue
Block a user