[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -3,7 +3,9 @@
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
@@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _traffic_loop(
|
||||
server: RemoteOpenAIServer,
|
||||
dp_rank: int | None,
|
||||
ready: threading.Barrier,
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
request_payload = None if is_probe else payload
|
||||
responses = []
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code))
|
||||
if status_code == 200:
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
if finished.is_set():
|
||||
return responses
|
||||
time.sleep(0.05)
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
|
||||
rejected = [end for _, end, status in responses if status == 503]
|
||||
if not rejected:
|
||||
return 0
|
||||
recovered = next(
|
||||
end for _, end, status in responses if status == 200 and end > rejected[-1]
|
||||
)
|
||||
return recovered - rejected[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
server: RemoteOpenAIServer,
|
||||
source_dp_size: int,
|
||||
new_dp_size: int,
|
||||
traffic_mode: str,
|
||||
) -> None:
|
||||
traffic_clients: list[int | None] = []
|
||||
if traffic_mode == "light":
|
||||
traffic_clients = [0]
|
||||
elif traffic_mode == "heavy":
|
||||
traffic_clients = [None] * source_dp_size
|
||||
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
|
||||
ready = threading.Barrier(len(clients) + 1)
|
||||
stop = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
|
||||
futures = [
|
||||
executor.submit(
|
||||
_traffic_loop, server, rank, ready, stop, finished, is_probe
|
||||
)
|
||||
for rank, is_probe in clients
|
||||
]
|
||||
try:
|
||||
ready.wait(timeout=120)
|
||||
start_time = time.perf_counter()
|
||||
assert _send_scale_command(server, new_dp_size)
|
||||
scale_seconds = time.perf_counter() - start_time
|
||||
finished.set()
|
||||
probe_result, *results = [future.result(timeout=120) for future in futures]
|
||||
finally:
|
||||
stop.set()
|
||||
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_503 = [start for start, _, status in probe_result if status == 503]
|
||||
assert probe_503, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200 and start_time <= request_start and request_end < probe_503[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
|
||||
f"[traffic={traffic_mode}] "
|
||||
f"scale_seconds={scale_seconds:.3f} "
|
||||
f"downtime_seconds={_downtime(probe_result):.3f}"
|
||||
)
|
||||
|
||||
|
||||
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
assert server.port is not None
|
||||
result = evaluate_gsm8k(
|
||||
@@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
return accuracy
|
||||
|
||||
|
||||
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
@@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"true" if use_async_eplb else "false",
|
||||
"true",
|
||||
"--eplb-config.step_interval",
|
||||
"10",
|
||||
"300",
|
||||
"--eplb-config.window_size",
|
||||
"5",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
"2",
|
||||
str(dp_size),
|
||||
"--api-server-count",
|
||||
"1",
|
||||
"--disable-access-log-for-endpoints",
|
||||
"/is_scaling_elastic_ep",
|
||||
]
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
args.extend(["--data-parallel-address", leader_address])
|
||||
if enforce_eager:
|
||||
args.append("--enforce-eager")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
("enforce_eager", "traffic_mode"),
|
||||
[
|
||||
pytest.param(True, "none", id="enforce_eager_none"),
|
||||
pytest.param(True, "light", id="enforce_eager_light"),
|
||||
pytest.param(True, "heavy", id="enforce_eager_heavy"),
|
||||
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
|
||||
],
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
|
||||
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
|
||||
assert target_dp_size > initial_dp_size
|
||||
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
) as server:
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
|
||||
|
||||
assert _send_scale_command(server, 4)
|
||||
time.sleep(10)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)")
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial")
|
||||
|
||||
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
assert _send_scale_command(server, 2)
|
||||
time.sleep(5)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
|
||||
|
||||
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
@@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
def test_elastic_ep_scaling_uneven():
|
||||
"""Test scale up with uneven worker distribution.
|
||||
|
||||
This tests the case where num_new_workers % old_dp_size != 0,
|
||||
specifically 2 -> 3 where remainder = 1 % 2 = 1.
|
||||
This exercises the remainder handling in sender-receiver pairing.
|
||||
"""
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
vllm_serve_args = _base_serve_args()
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
@@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
|
||||
# Tests uneven sender-receiver pairing
|
||||
assert _send_scale_command(server, 3)
|
||||
time.sleep(10)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
|
||||
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
@@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
|
||||
# Scale back down to 2
|
||||
assert _send_scale_command(server, 2)
|
||||
time.sleep(5)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
|
||||
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
|
||||
Reference in New Issue
Block a user