[Elastic EP] Async preparation (#47288)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-07-28 05:19:13 -07:00
committed by GitHub
parent d2bfc6fe20
commit 35efdf6b34
21 changed files with 701 additions and 677 deletions
+135 -31
View File
@@ -3,7 +3,9 @@
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import pytest
import requests
@@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
return False
def _traffic_loop(
server: RemoteOpenAIServer,
dp_rank: int | None,
ready: threading.Barrier,
stop: threading.Event,
finished: threading.Event,
is_probe: bool = False,
) -> list[tuple[float, float, int | None]]:
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
request_payload = None if is_probe else payload
responses = []
is_ready = False
while not stop.is_set():
request_start = time.perf_counter()
try:
response = requests.post(
url, json=request_payload, headers=headers, timeout=120
)
status_code = response.status_code
except requests.exceptions.RequestException:
status_code = None
responses.append((request_start, time.perf_counter(), status_code))
if status_code == 200:
if not is_ready:
ready.wait(timeout=120)
is_ready = True
if finished.is_set():
return responses
time.sleep(0.05)
return responses
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
rejected = [end for _, end, status in responses if status == 503]
if not rejected:
return 0
recovered = next(
end for _, end, status in responses if status == 200 and end > rejected[-1]
)
return recovered - rejected[0]
def _scale_with_traffic(
server: RemoteOpenAIServer,
source_dp_size: int,
new_dp_size: int,
traffic_mode: str,
) -> None:
traffic_clients: list[int | None] = []
if traffic_mode == "light":
traffic_clients = [0]
elif traffic_mode == "heavy":
traffic_clients = [None] * source_dp_size
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
ready = threading.Barrier(len(clients) + 1)
stop = threading.Event()
finished = threading.Event()
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
futures = [
executor.submit(
_traffic_loop, server, rank, ready, stop, finished, is_probe
)
for rank, is_probe in clients
]
try:
ready.wait(timeout=120)
start_time = time.perf_counter()
assert _send_scale_command(server, new_dp_size)
scale_seconds = time.perf_counter() - start_time
finished.set()
probe_result, *results = [future.result(timeout=120) for future in futures]
finally:
stop.set()
bad_statuses = {
status
for responses in [probe_result, *results]
for _, _, status in responses
if status not in (200, 503)
}
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
probe_503 = [start for start, _, status in probe_result if status == 503]
assert probe_503, "Scaling probe did not observe commit"
assert not results or any(
status == 200 and start_time <= request_start and request_end < probe_503[0]
for responses in results
for request_start, request_end, status in responses
), "No request completed successfully during preparation"
print(
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
f"[traffic={traffic_mode}] "
f"scale_seconds={scale_seconds:.3f} "
f"downtime_seconds={_downtime(probe_result):.3f}"
)
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
assert server.port is not None
result = evaluate_gsm8k(
@@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
return accuracy
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
args = [
"--trust-remote-code",
"--tensor-parallel-size",
@@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
"--eplb-config.num_redundant_experts",
"0",
"--eplb-config.use_async",
"true" if use_async_eplb else "false",
"true",
"--eplb-config.step_interval",
"10",
"300",
"--eplb-config.window_size",
"5",
"--data-parallel-backend",
"ray",
"--data-parallel-size",
"2",
str(dp_size),
"--api-server-count",
"1",
"--disable-access-log-for-endpoints",
"/is_scaling_elastic_ep",
]
leader_address = os.environ.get("LEADER_ADDRESS")
if leader_address:
args.extend(["--data-parallel-address", leader_address])
if enforce_eager:
args.append("--enforce-eager")
return args
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
("enforce_eager", "traffic_mode"),
[
pytest.param(True, "none", id="enforce_eager_none"),
pytest.param(True, "light", id="enforce_eager_light"),
pytest.param(True, "heavy", id="enforce_eager_heavy"),
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
],
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling(use_async_eplb: bool):
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
assert target_dp_size > initial_dp_size
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
) as server:
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
assert _send_scale_command(server, 4)
time.sleep(10)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)")
initial_accuracy = _run_gsm8k_eval(server, "Initial")
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
assert _send_scale_command(server, 2)
time.sleep(5)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
@@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool):
print(f" Tolerance: {ACCURACY_TOL:.3f}")
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
def test_elastic_ep_scaling_uneven():
"""Test scale up with uneven worker distribution.
This tests the case where num_new_workers % old_dp_size != 0,
specifically 2 -> 3 where remainder = 1 % 2 = 1.
This exercises the remainder handling in sender-receiver pairing.
"""
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
vllm_serve_args = _base_serve_args()
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
@@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
# Tests uneven sender-receiver pairing
assert _send_scale_command(server, 3)
time.sleep(10)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
@@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
# Scale back down to 2
assert _send_scale_command(server, 2)
time.sleep(5)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (