[Elastic EP] Async preparation (#47288)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-07-28 05:19:13 -07:00
committed by GitHub
parent d2bfc6fe20
commit 35efdf6b34
21 changed files with 701 additions and 677 deletions
@@ -52,4 +52,5 @@ steps:
- vllm/compilation/
- tests/distributed/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- pytest -v -s distributed/test_elastic_ep.py
+135 -31
View File
@@ -3,7 +3,9 @@
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import pytest
import requests
@@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
return False
def _traffic_loop(
server: RemoteOpenAIServer,
dp_rank: int | None,
ready: threading.Barrier,
stop: threading.Event,
finished: threading.Event,
is_probe: bool = False,
) -> list[tuple[float, float, int | None]]:
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
request_payload = None if is_probe else payload
responses = []
is_ready = False
while not stop.is_set():
request_start = time.perf_counter()
try:
response = requests.post(
url, json=request_payload, headers=headers, timeout=120
)
status_code = response.status_code
except requests.exceptions.RequestException:
status_code = None
responses.append((request_start, time.perf_counter(), status_code))
if status_code == 200:
if not is_ready:
ready.wait(timeout=120)
is_ready = True
if finished.is_set():
return responses
time.sleep(0.05)
return responses
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
rejected = [end for _, end, status in responses if status == 503]
if not rejected:
return 0
recovered = next(
end for _, end, status in responses if status == 200 and end > rejected[-1]
)
return recovered - rejected[0]
def _scale_with_traffic(
server: RemoteOpenAIServer,
source_dp_size: int,
new_dp_size: int,
traffic_mode: str,
) -> None:
traffic_clients: list[int | None] = []
if traffic_mode == "light":
traffic_clients = [0]
elif traffic_mode == "heavy":
traffic_clients = [None] * source_dp_size
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
ready = threading.Barrier(len(clients) + 1)
stop = threading.Event()
finished = threading.Event()
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
futures = [
executor.submit(
_traffic_loop, server, rank, ready, stop, finished, is_probe
)
for rank, is_probe in clients
]
try:
ready.wait(timeout=120)
start_time = time.perf_counter()
assert _send_scale_command(server, new_dp_size)
scale_seconds = time.perf_counter() - start_time
finished.set()
probe_result, *results = [future.result(timeout=120) for future in futures]
finally:
stop.set()
bad_statuses = {
status
for responses in [probe_result, *results]
for _, _, status in responses
if status not in (200, 503)
}
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
probe_503 = [start for start, _, status in probe_result if status == 503]
assert probe_503, "Scaling probe did not observe commit"
assert not results or any(
status == 200 and start_time <= request_start and request_end < probe_503[0]
for responses in results
for request_start, request_end, status in responses
), "No request completed successfully during preparation"
print(
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
f"[traffic={traffic_mode}] "
f"scale_seconds={scale_seconds:.3f} "
f"downtime_seconds={_downtime(probe_result):.3f}"
)
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
assert server.port is not None
result = evaluate_gsm8k(
@@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
return accuracy
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
args = [
"--trust-remote-code",
"--tensor-parallel-size",
@@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
"--eplb-config.num_redundant_experts",
"0",
"--eplb-config.use_async",
"true" if use_async_eplb else "false",
"true",
"--eplb-config.step_interval",
"10",
"300",
"--eplb-config.window_size",
"5",
"--data-parallel-backend",
"ray",
"--data-parallel-size",
"2",
str(dp_size),
"--api-server-count",
"1",
"--disable-access-log-for-endpoints",
"/is_scaling_elastic_ep",
]
leader_address = os.environ.get("LEADER_ADDRESS")
if leader_address:
args.extend(["--data-parallel-address", leader_address])
if enforce_eager:
args.append("--enforce-eager")
return args
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
("enforce_eager", "traffic_mode"),
[
pytest.param(True, "none", id="enforce_eager_none"),
pytest.param(True, "light", id="enforce_eager_light"),
pytest.param(True, "heavy", id="enforce_eager_heavy"),
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
],
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling(use_async_eplb: bool):
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
assert target_dp_size > initial_dp_size
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
) as server:
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
assert _send_scale_command(server, 4)
time.sleep(10)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)")
initial_accuracy = _run_gsm8k_eval(server, "Initial")
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
assert _send_scale_command(server, 2)
time.sleep(5)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
@@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool):
print(f" Tolerance: {ACCURACY_TOL:.3f}")
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
def test_elastic_ep_scaling_uneven():
"""Test scale up with uneven worker distribution.
This tests the case where num_new_workers % old_dp_size != 0,
specifically 2 -> 3 where remainder = 1 % 2 = 1.
This exercises the remainder handling in sender-receiver pairing.
"""
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
vllm_serve_args = _base_serve_args()
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
@@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
# Tests uneven sender-receiver pairing
assert _send_scale_command(server, 3)
time.sleep(10)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
@@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
# Scale back down to 2
assert _send_scale_command(server, 2)
time.sleep(5)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
+9
View File
@@ -686,6 +686,14 @@ class ParallelConfig:
and self.data_parallel_size > 1
)
@property
def use_all2all(self) -> bool:
return (
self.data_parallel_size > 1
or self.use_sequence_parallel_moe
or (self.enable_expert_parallel and self.prefill_context_parallel_size > 1)
)
@property
def use_batched_dp_moe(self) -> bool:
return (
@@ -786,6 +794,7 @@ class ParallelConfig:
"data_parallel_master_ip",
"data_parallel_master_port",
"_data_parallel_master_port_list",
"_coord_store_port",
"data_parallel_rpc_port",
"rank",
"master_addr",
@@ -175,6 +175,7 @@ class DeviceCommunicatorBase:
unique_name: str = "",
global_ranks: list[int] | None = None,
global_world_size: int | None = None,
use_all2all: bool = False,
):
self.device = device or torch.device("cpu")
self.cpu_group = cpu_group
@@ -204,26 +205,15 @@ class DeviceCommunicatorBase:
self.global_world_size = dist.get_world_size()
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
use_ep = False
all2all_backend = None
from vllm.config import get_current_vllm_config_or_none
config = get_current_vllm_config_or_none()
if config is not None:
# initialize the all2all manager for DP or sequence-parallel EP.
parallel_config = config.parallel_config
use_ep = (
parallel_config.data_parallel_size > 1
or parallel_config.use_sequence_parallel_moe
or (
parallel_config.enable_expert_parallel
and parallel_config.prefill_context_parallel_size > 1
)
)
all2all_backend = parallel_config.all2all_backend
all2all_backend = config.parallel_config.all2all_backend
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
self.use_all2all = self.is_ep_communicator and use_ep
self.use_all2all = self.is_ep_communicator and use_all2all
self.all2all_backend = all2all_backend
self.all2all_manager: All2AllManagerBase | None = None
@@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase):
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(cpu_group, device, device_group, unique_name)
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.dist_module = torch.distributed
if (
@@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
global_ranks: list[int] | None = None,
global_world_size: int | None = None,
tcp_store_group: StatelessProcessGroup | None = None,
use_all2all: bool = False,
):
super().__init__(
cpu_group,
@@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
unique_name,
global_ranks,
global_world_size,
use_all2all=use_all2all,
)
if "tp" not in unique_name:
# custom allreduce or torch symm mem can be used only by tp
@@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase):
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(cpu_group, device, device_group, unique_name)
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.ca_comm: None = None
if self.use_all2all:
if self.all2all_backend in ("naive", "allgather_reducescatter"):
+95 -86
View File
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import gc
import weakref
from collections.abc import Iterable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import replace
from typing import TYPE_CHECKING
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
make_eep_staged_quant_method,
)
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.utils import is_moe_layer
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
@@ -145,6 +146,10 @@ class ElasticEPScalingExecutor:
self.worker_ref = weakref.ref(worker)
self.reconfig_request = None
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
self._async_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="ElasticEPAsync"
)
self._async_future: Future[None] | None = None
@property
def worker(self):
@@ -159,59 +164,68 @@ class ElasticEPScalingExecutor:
raise ValueError(f"Unknown execute method: {execute_method}")
return method(*args, **kwargs)
def _set_eplb_suppressed(self, suppressed: bool) -> None:
self.worker.model_runner.eep_eplb_suppressed = suppressed
ep_group = get_standby_ep_group() or get_ep_group()
if ep_group.rank == 0:
logger.info(
"[Elastic EP] EPLB %s elastic scaling transition",
"disabled during" if suppressed else "re-enabled after",
)
def start_async(self, execute_method: str, *args, **kwargs) -> str:
if self._async_future is not None:
raise RuntimeError("Another Elastic EP async method is active")
if args and isinstance(args[0], ReconfigureDistributedRequest):
self.reconfig_request = args[0]
dp_rank = self.worker.vllm_config.parallel_config.data_parallel_rank
done_key = f"eep_async/{execute_method}/{dp_rank}/{self.worker.rank}"
self._async_future = self._async_executor.submit(
self._run_async, execute_method, *args, **kwargs
)
self._async_future.add_done_callback(lambda _: self._mark_async_done(done_key))
return done_key
def _run_async(self, execute_method: str, *args, **kwargs) -> None:
from vllm.platforms import current_platform
self.worker.vllm_config.enable_trace_function_call_for_thread()
assert hasattr(self.worker, "device")
current_platform.set_device(self.worker.device)
with set_current_vllm_config(self.worker.vllm_config):
self.execute(execute_method, *args, **kwargs)
def _mark_async_done(self, done_key: str) -> None:
from vllm.distributed.utils import get_cached_tcp_store_client
assert self.reconfig_request is not None
get_cached_tcp_store_client(
self.reconfig_request.new_data_parallel_master_ip,
self.reconfig_request.coord_store_port,
).set(done_key, b"1")
def clear_async(self) -> None:
future = self._async_future
if future is None:
raise RuntimeError("No Elastic EP async method is active")
if not future.done():
raise RuntimeError("Elastic EP async method is not done")
self._async_future = None
future.result()
def load_model(self) -> None:
(
expanded_physical_to_logical,
num_logical_experts,
old_num_physical_experts,
) = self.receive_expert_mapping()
num_physical_experts = expanded_physical_to_logical.shape[1]
self.worker.parallel_config.eplb_config.num_redundant_experts = (
num_physical_experts - num_logical_experts
)
self.worker.load_model(load_dummy_weights=True)
self.worker.model_runner.setup_eplb_from_mapping(
expanded_physical_to_logical, old_num_physical_experts
)
self._set_eplb_suppressed(True)
def create_standby_groups(
self, reconfig_request: ReconfigureDistributedRequest
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
) -> None:
self.reconfig_request = reconfig_request
new_dp_size = reconfig_request.new_data_parallel_size
old_dp_size = get_dp_group().world_size
world_size = self.worker.vllm_config.parallel_config.world_size
parallel_config = self.worker.vllm_config.parallel_config
world_size = parallel_config.world_size
new_world_size_across_dp = world_size * new_dp_size
updated_config = copy.copy(self.worker.vllm_config)
updated_config.parallel_config = copy.deepcopy(
self.worker.vllm_config.parallel_config
create_standby_groups(
new_dp_size=new_dp_size,
new_world_size_across_dp=new_world_size_across_dp,
master_ip=reconfig_request.new_data_parallel_master_ip,
coord_store_port=reconfig_request.coord_store_port,
use_all2all=use_all2all,
enable_eplb=parallel_config.enable_eplb,
)
updated_config.parallel_config.data_parallel_size = new_dp_size
with set_current_vllm_config(updated_config):
create_standby_groups(
new_dp_size=new_dp_size,
new_world_size_across_dp=new_world_size_across_dp,
master_ip=reconfig_request.new_data_parallel_master_ip,
coord_store_port=reconfig_request.coord_store_port,
enable_eplb=updated_config.parallel_config.enable_eplb,
)
if new_dp_size > old_dp_size:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
elif new_dp_size < old_dp_size:
self._stage_standby_moe_quant_methods()
if new_dp_size < old_dp_size:
self.stage_standby_moe_quant_methods()
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
standby_dp_group = get_standby_dp_group()
@@ -265,6 +279,7 @@ class ElasticEPScalingExecutor:
model_config = self.worker.model_runner.model_config
eplb_state = self.worker.model_runner.eplb_state
assert eplb_state is not None
eplb_state.drain_async()
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
physical_to_logical = eplb_model_state.physical_to_logical_map
num_physical_experts = physical_to_logical.shape[1]
@@ -278,10 +293,6 @@ class ElasticEPScalingExecutor:
src_rank=0,
device=self.worker.device,
)
# New workers enter load_model after receiving the expert mapping.
# Stage replacement MoE kernels before returning to the state machine
# so existing ranks can participate in collective EP comm creation.
self._stage_standby_moe_quant_methods()
def _make_eep_moe_config(self, module, dp_group, ep_group):
parallel_config = self.worker.vllm_config.parallel_config
@@ -300,7 +311,7 @@ class ElasticEPScalingExecutor:
moe_parallel_config=moe_parallel_config,
)
def _stage_standby_moe_quant_methods(self) -> None:
def stage_standby_moe_quant_methods(self) -> None:
standby_dp_group = get_standby_dp_group()
standby_ep_group = get_standby_ep_group()
model = self.worker.model_runner.get_model()
@@ -500,26 +511,6 @@ class ElasticEPScalingExecutor:
compilation_counter.stock_torch_compile_count += 1
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
multi_block_table = self.worker.model_runner.input_batch.block_table
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
saved_block_tables.append(
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
)
multi_block_table.clear()
unlock_workspace()
self.worker.compile_or_warm_up_model()
lock_workspace()
for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved_block_tables
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
if new_dp_size < old_dp_size:
self._set_eplb_suppressed(False)
def _perform_eplb_reshuffle(
self, rank_mapping: dict[int, int] | None = None
) -> None:
@@ -553,12 +544,25 @@ class ElasticEPScalingExecutor:
if get_ep_group().rank == 0:
logger.info("[Elastic EP] Expert resharding completed")
def perform_eplb_reshuffle(self) -> None:
def commit_scale_up(self, is_existing_worker: bool) -> None:
if is_existing_worker:
self.broadcast_expert_mapping()
self.switch_and_prepare()
else:
mapping, _, num_valid_experts = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
self._perform_eplb_reshuffle()
self._set_eplb_suppressed(False)
self.warm_and_capture()
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
self.perform_scale_down_eplb_reshuffle(new_dp_size)
if removing:
self.switch_and_remove()
else:
self.switch_and_prepare()
self.warm_and_capture()
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
@@ -599,12 +603,17 @@ class ElasticEPScalingExecutor:
)
model = self.worker.model_runner.get_model()
expert_weights = [
module.get_expert_weights()
for module in model.modules()
if is_moe_layer(module)
]
batch_transfer_weights(
model=model,
is_sender=False,
peer_rank=sender_rank,
dp_group=dp_group,
expert_weights=model.expert_weights,
expert_weights=expert_weights,
)
torch.accelerator.synchronize()
@@ -643,14 +652,17 @@ class ElasticEPScalingExecutor:
with set_current_vllm_config(self.worker.vllm_config):
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
def rewarm_workspace(self) -> None:
def warmup_local_kernels(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):
kernel_warmup(self.worker, process_local_only=True)
def warm_and_capture(self) -> None:
# Must run on every DP sibling in lockstep: _dummy_run calls
# coordinate_batch_across_dp whenever data_parallel_size > 1
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
# Save and clear block tables so profile_run/compile_or_warm_up_model
# don't write dummy slot mappings into real KV-cache blocks (mirrors
# switch_and_prepare's pattern).
# Save and clear block tables so the dummy MoE forward doesn't
# write dummy slot mappings into real KV-cache blocks.
multi_block_table = self.worker.model_runner.input_batch.block_table
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
@@ -660,19 +672,16 @@ class ElasticEPScalingExecutor:
multi_block_table.clear()
# _ensure_workspace_size allocates a fresh tensor on grow, leaving
# captured CUDA graphs with stale data pointers; drop graphs before
# re-warm so captures realign with the resized buffer.
# any captured CUDA graph with a stale data pointer; drop graphs
# before re-warm so captures realign with the resized buffer.
self._release_cuda_graphs()
unlock_workspace()
# Grow the MoE workspace at max_num_tokens.
# compile_or_warm_up_model alone only exercises cudagraph-capture
# sizes (≤64 tokens for this test) and leaves the workspace at
# ~10-14 MB; the post-all-to-all per-rank token count under real
# post-reshuffle routing needs hundreds of MB. Use _dummy_run
# directly (rather than profile_run) with skip_eplb=True so dummy
# routing doesn't pollute the just-rebalanced EPLB stats — same
# convention compile_or_warm_up_model itself uses.
# Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model
# alone only exercises cudagraph-capture sizes and can leave the
# workspace too small for post-reshuffle routing. Use _dummy_run
# directly with skip_eplb=True so dummy routing doesn't pollute the
# just-rebalanced EPLB stats.
runner = self.worker.model_runner
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
self.worker.compile_or_warm_up_model()
+162 -298
View File
@@ -1,18 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import enum
import time
import weakref
from datetime import timedelta
from typing import TYPE_CHECKING, Literal, TypeAlias
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
import torch.distributed
from vllm.config import ParallelConfig
from vllm.distributed import (
sched_yield,
stateless_destroy_torch_distributed_process_group,
)
from vllm.distributed.utils import get_cached_tcp_store_client
from vllm.logger import init_logger
from vllm.v1.engine import (
EEPNotificationType,
@@ -31,35 +30,29 @@ WorkerType = Literal["existing", "new", "removing"]
class ScaleUpExistingEngineState(enum.IntEnum):
WAIT_NEW_CORE_ENGINES_INIT = 0
CREATE_STANDBY_GROUPS = 1
TRANSFER_EXPERT_MAPPING = 2
WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT = 3
TRANSFER_WEIGHTS = 4
SYNC_KV_CACHE_MEMORY_SIZE = 5
SWITCH_AND_PREPARE = 6
EPLB_RESHUFFLE = 7
COMPLETE = 8
CREATE_STANDBY_GROUPS = 0
STAGE_QUANT_METHODS = 1
TRANSFER_WEIGHTS = 2
SYNC_KV_CACHE_MEMORY_SIZE = 3
COMMIT_SCALE_UP = 4 # Blocks forward passes.
COMPLETE = 5
class ScaleUpNewEngineState(enum.IntEnum):
PRE_KV_INIT = 0
PREPARE = 1
EPLB_RESHUFFLE = 2
COMPLETE = 3
COMPLETE = 2
class ScaleDownRemainingEngineState(enum.IntEnum):
PREPARE = 0
EPLB_RESHUFFLE = 1
SWITCH_AND_PREPARE = 2
COMPLETE = 3
COMMIT_SCALE_DOWN = 1 # Blocks forward passes.
COMPLETE = 2
class ScaleDownRemovingEngineState(enum.IntEnum):
PREPARE = 0
EPLB_RESHUFFLE = 1
COMPLETE = 2
COMPLETE = 1
EngineState: TypeAlias = (
@@ -70,15 +63,6 @@ EngineState: TypeAlias = (
)
class _BarrierTimeoutError(RuntimeError):
"""
Exception raised for timeout
in the first stage of our two-staged
TCPStore based barrier to synchronize the
execution of all engines in the DP group.
"""
class ElasticEPScalingState:
def __init__(
self,
@@ -94,20 +78,24 @@ class ElasticEPScalingState:
self.engine_core_ref = weakref.ref(engine_core)
self.vllm_config = vllm_config
self.old_dp_group = self.engine_core.dp_group if worker_type != "new" else None
self.old_dp_store = self.engine_core.dp_store if worker_type != "new" else None
self.new_parallel_config: ParallelConfig = new_parallel_config
self.new_dp_group = self.engine_core.dp_group if worker_type == "new" else None
self.new_dp_store = self.engine_core.dp_store if worker_type == "new" else None
self.worker_type = worker_type
self.scale_type = scale_type
self.reconfig_request = reconfig_request
self.commit_requested = False
self._prepare_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="ElasticEPPrepare"
)
self._prepare_future: Future[Any] | None = None
self._new_dp_sync: tuple[object, Any] | None = None
self.state: EngineState
if scale_type == "scale_up":
self.state = (
ScaleUpNewEngineState.PRE_KV_INIT
if worker_type == "new"
else ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
)
else:
self.state = (
@@ -130,6 +118,31 @@ class ElasticEPScalingState:
raise RuntimeError("Engine core has been garbage collected")
return engine_core
def _collective_rpc(self, *args, **kwargs):
return self.model_executor.collective_rpc(*args, **kwargs)
def _execute_async(self, execute_method: str, *args) -> bool:
if self._prepare_future is None:
done_keys = self._collective_rpc(
"elastic_ep_execute",
args=("start_async", execute_method, *args),
)
assert self.reconfig_request is not None
coord_store = get_cached_tcp_store_client(
self.reconfig_request.new_data_parallel_master_ip,
self.reconfig_request.coord_store_port,
)
self._prepare_future = self._prepare_executor.submit(
coord_store.wait, done_keys
)
if not self._prepare_future.done():
return False
self._prepare_future.result()
self._collective_rpc("elastic_ep_execute", args=("clear_async",))
self._prepare_future = None
return True
def progress(self) -> bool:
if self.scale_type == "scale_up":
return (
@@ -149,157 +162,43 @@ class ElasticEPScalingState:
assert self.progress()
assert self.state == ScaleUpNewEngineState.PREPARE
def _execute_tcp_store_barrier(
self, dp_store, group_rank, group_size, barrier_id, timeout=None
):
arrival_key = f"arrival_{barrier_id}_{group_rank}"
dp_store.set(arrival_key, b"1")
start_time = time.time()
processes_arrived: set[int] = set()
while len(processes_arrived) < group_size:
if (
timeout is not None
and time.time() - start_time > timeout.total_seconds()
):
raise _BarrierTimeoutError(
f"Barrier timed out after {timeout.total_seconds()} seconds"
)
for i in range(group_size):
if i in processes_arrived:
continue
key = f"arrival_{barrier_id}_{i}"
present = dp_store.check([key])
if present:
processes_arrived.add(i)
if len(processes_arrived) < group_size:
sched_yield()
def _staged_barrier(self, use_new_group: bool, barrier_name: str) -> bool:
"""
Execute a two-staged barrier to synchronize all engines in the DP group.
Some DP EngineCores may receive the reconfiguration notifications
later than others, and already proceed to engine step (model forward)
in the busy loop.
In this case, EngineCores that already proceed to reconfiguration
should skip reconfiguration and execute model forward for one more
step, so in the next step, all EngineCores will be synchronized.
We use a two-staged barrier to achieve this. The first time each
EngineCore executes the barrier, if a timeout is reached before the
barrier completes, that means some EngineCores have already entered
engine step. The EngineCores that timed out will then proceed to
engine step, and will synchronize with the other EngineCores in the
next step with a barrier without timeout.
"""
dp_group = self.new_dp_group if use_new_group else self.old_dp_group
dp_store = self.new_dp_store if use_new_group else self.old_dp_store
assert dp_group is not None and dp_store is not None
group_rank = dp_group.rank()
group_size = dp_group.size()
barrier_id = f"eep_barrier_{barrier_name}"
sync_key = f"{barrier_id}_sync"
# TODO(yongji): figure out appropriate timeout for the barrier
timeout = None if dp_store.check([sync_key]) else timedelta(seconds=5)
try:
self._execute_tcp_store_barrier(
dp_store, group_rank, group_size, barrier_id, timeout=timeout
)
torch.distributed.barrier(dp_group)
if group_rank == 0:
dp_store.delete_key(sync_key)
for i in range(group_size):
dp_store.delete_key(f"arrival_{barrier_id}_{i}")
return True
except _BarrierTimeoutError as e:
if timeout is None:
raise RuntimeError("Unexpected timeout encountered") from e
dp_store.compare_set(sync_key, "", b"1")
return False
def _progress_existing_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT:
return False
elif state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
# NOTE(yongji): wait for all existing workers to receive the request
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
if not self._create_standby_groups():
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="create_standby_groups"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._create_standby_groups()
self.state = ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
return True
elif state == ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING:
self._transfer_expert_mapping()
self.state = ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
if not self._execute_async("stage_standby_moe_quant_methods"):
return False
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
return True
elif state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT:
return False
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
if not self._transfer_weights():
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="transfer_weights"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._transfer_weights()
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
return True
elif state == ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE:
self._sync_kv_cache_memory_size()
self.state = ScaleUpExistingEngineState.SWITCH_AND_PREPARE
if not self._sync_kv_cache_memory_size():
return False
self.state = ScaleUpExistingEngineState.COMMIT_SCALE_UP
self._mark_ready_for_switch()
return True
elif state == ScaleUpExistingEngineState.SWITCH_AND_PREPARE:
self._switch_and_prepare()
self.state = ScaleUpExistingEngineState.EPLB_RESHUFFLE
assert self.new_dp_store is not None
self.new_dp_store.add("eep_barrier_engine_count", 1)
return True
elif state == ScaleUpExistingEngineState.EPLB_RESHUFFLE:
assert self.new_dp_group is not None and self.new_dp_store is not None
if (
int(self.new_dp_store.get("eep_barrier_engine_count"))
< self.new_dp_group.size()
):
elif state == ScaleUpExistingEngineState.COMMIT_SCALE_UP:
if not self.commit_requested:
return False
if not self._staged_barrier(
use_new_group=True, barrier_name="eplb_reshuffle"
):
return False
if self.new_dp_group.rank() == 0:
self.new_dp_store.delete_key("eep_barrier_engine_count")
self._eplb_reshuffle()
self._commit_new_dp_group()
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", True))
self.state = ScaleUpExistingEngineState.COMPLETE
self._update_parallel_config()
self._send_reconfigure_finished()
return True
else:
@@ -311,22 +210,17 @@ class ElasticEPScalingState:
assert self.new_dp_group is not None and self.new_dp_store is not None
if state == ScaleUpNewEngineState.PRE_KV_INIT:
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("receive_weights",)
)
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
self.engine_core.available_gpu_memory_for_kv_cache = (
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("prepare_new_worker",)
)
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
self.state = ScaleUpNewEngineState.PREPARE
return True
elif state == ScaleUpNewEngineState.PREPARE:
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
self._mark_ready_for_switch()
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(
tensor,
@@ -337,22 +231,7 @@ class ElasticEPScalingState:
self.engine_core.engines_running = bool(data[0])
self.engine_core.current_wave = int(data[1])
self.engine_core.step_counter = int(data[2])
self.state = ScaleUpNewEngineState.EPLB_RESHUFFLE
self.new_dp_store.add("eep_barrier_engine_count", 1)
return True
elif state == ScaleUpNewEngineState.EPLB_RESHUFFLE:
if (
int(self.new_dp_store.get("eep_barrier_engine_count"))
< self.new_dp_group.size()
):
return False
if not self._staged_barrier(
use_new_group=True, barrier_name="eplb_reshuffle"
):
return False
assert self.new_dp_group.rank() > 0
self._eplb_reshuffle()
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", False))
self.state = ScaleUpNewEngineState.COMPLETE
return True
@@ -362,38 +241,23 @@ class ElasticEPScalingState:
def _progress_remaining_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleDownRemainingEngineState.PREPARE:
self.state = ScaleDownRemainingEngineState.EPLB_RESHUFFLE
self.old_dp_store.add("eep_barrier_engine_count", 1)
return True
if self._create_standby_groups():
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
self._mark_ready_for_switch()
return True
return False
elif state == ScaleDownRemainingEngineState.EPLB_RESHUFFLE:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
elif state == ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN:
if not self.commit_requested:
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="eplb_reshuffle"
):
return False
if self.old_dp_group.rank() == 0:
self.old_dp_store.delete_key("eep_barrier_engine_count")
self._eplb_reshuffle_before_scale_down()
self.state = ScaleDownRemainingEngineState.SWITCH_AND_PREPARE
# NOTE(yongji): currently, after EPLB reshuffle
# that redistributes experts to remaining workers, workers
# to be removed will immediately initiate shutdown;
# existing workers can no longer execute forward steps using
# the old setup. In the future, we may keep
# the removing workers alive a bit longer,
# e.g., to drain in-batch requests.
self._create_standby_groups()
self._switch_and_prepare()
self._commit_scale_down(removing=False)
self._commit_new_dp_group()
self._update_parallel_config()
self.state = ScaleDownRemainingEngineState.COMPLETE
self._send_reconfigure_finished()
return True
else:
@@ -402,26 +266,11 @@ class ElasticEPScalingState:
def _progress_removing_engine(self) -> bool:
state = self.state
assert self.old_dp_group is not None and self.old_dp_store is not None
assert self.old_dp_group is not None
if state == ScaleDownRemovingEngineState.PREPARE:
self.state = ScaleDownRemovingEngineState.EPLB_RESHUFFLE
self.old_dp_store.add("eep_barrier_engine_count", 1)
return True
if state == ScaleDownRemovingEngineState.EPLB_RESHUFFLE:
if (
int(self.old_dp_store.get("eep_barrier_engine_count"))
< self.old_dp_group.size()
):
return False
if not self._staged_barrier(
use_new_group=False, barrier_name="eplb_reshuffle"
):
return False
assert self.old_dp_group.rank() > 0
self._eplb_reshuffle_before_scale_down()
self._switch_and_remove()
self._commit_scale_down(removing=True)
self.state = ScaleDownRemovingEngineState.COMPLETE
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.SHUTDOWN_COMPLETE
@@ -432,22 +281,22 @@ class ElasticEPScalingState:
assert self.state == ScaleDownRemovingEngineState.COMPLETE
return True
def handle_notification(self, notification_type: EEPNotificationType):
assert self.worker_type != "new"
assert self.old_dp_store is not None
if (
notification_type == EEPNotificationType.NEW_CORE_ENGINES_INIT_READY
and self.state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
):
self.old_dp_store.add("eep_barrier_engine_count", 1)
self.state = ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
elif (
notification_type == EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
and self.state
== ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
):
self.old_dp_store.add("eep_barrier_engine_count", 1)
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
def is_ready_for_switch(self) -> bool:
return self.worker_type == "existing" and (
self.state is ScaleUpExistingEngineState.COMMIT_SCALE_UP
or self.state is ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
)
@property
def ready_key(self) -> str:
return f"eep_ready/{self.engine_core.dp_rank}"
def _mark_ready_for_switch(self) -> None:
parallel_config = self.new_parallel_config
get_cached_tcp_store_client(
parallel_config.data_parallel_master_ip,
parallel_config._coord_store_port,
).set(self.ready_key, b"1")
def is_complete(self) -> bool:
if self.scale_type == "scale_up":
@@ -462,50 +311,78 @@ class ElasticEPScalingState:
else self.state == ScaleDownRemainingEngineState.COMPLETE
)
def _create_standby_groups(self):
def _init_new_dp_group(self) -> tuple[Any, Any]:
return self.new_parallel_config.stateless_init_dp_group(return_store=True)
def _ensure_new_dp_group(self) -> bool:
if self.new_dp_group is not None:
return True
if self._prepare_future is None:
self._prepare_future = self._prepare_executor.submit(
self._init_new_dp_group
)
if not self._prepare_future.done():
return False
self.new_dp_group, self.new_dp_store = self._prepare_future.result()
self._prepare_future = None
return True
def _create_standby_groups(self) -> bool:
assert self.old_dp_group is not None
self.new_dp_group, self.new_dp_store = (
self.new_parallel_config.stateless_init_dp_group(return_store=True)
)
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("create_standby_groups", self.reconfig_request)
)
if not self._ensure_new_dp_group():
return False
if not self._execute_async(
"create_standby_groups",
self.reconfig_request,
self.new_parallel_config.use_all2all,
):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Created standby communication groups")
return True
def _transfer_weights(self):
def _transfer_weights(self) -> bool:
assert self.reconfig_request is not None and self.old_dp_group is not None
old_dp_size = self.old_dp_group.size()
new_dp_size = self.reconfig_request.new_data_parallel_size
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("transfer_weights", old_dp_size, new_dp_size)
)
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Transferred weights to new workers")
return True
def _transfer_expert_mapping(self):
assert self.old_dp_group is not None
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("broadcast_expert_mapping",)
)
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Broadcasted expert mapping to new workers")
def _sync_kv_cache_memory_size(self):
def _sync_kv_cache_memory_size(self) -> bool:
assert self.engine_core.available_gpu_memory_for_kv_cache > 0
assert self.new_dp_group is not None and self.old_dp_group is not None
ParallelConfig.sync_kv_cache_memory_size(
self.new_dp_group,
self.engine_core.available_gpu_memory_for_kv_cache,
)
if self._new_dp_sync is None:
tensor = torch.tensor(
[self.engine_core.available_gpu_memory_for_kv_cache],
dtype=torch.int64,
device="cpu",
)
work = torch.distributed.all_reduce(
tensor,
op=torch.distributed.ReduceOp.MIN,
group=self.new_dp_group,
async_op=True,
)
self._new_dp_sync = (tensor, work)
return False
_, work = self._new_dp_sync
if not work.is_completed():
return False
work.wait()
self._new_dp_sync = None
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Synced KV cache memory size to new workers")
return True
def _switch_and_prepare(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("switch_and_prepare",)
)
def _commit_new_dp_group(self):
old_dp_group = self.old_dp_group
stateless_destroy_torch_distributed_process_group(old_dp_group)
assert self.new_dp_group is not None
@@ -529,41 +406,28 @@ class ElasticEPScalingState:
self.engine_core.current_wave = int(data[1])
self.engine_core.step_counter = int(data[2])
if new_dp_group.rank() == 0:
logger.info("[Elastic EP] Switched to new setup")
def _send_reconfigure_finished(self):
assert self.new_dp_group is not None
if self.new_dp_group.rank() == 0:
self.engine_core._eep_send_engine_core_notification(
EEPNotificationType.RECONFIGURE_FINISHED
)
logger.info("[Elastic EP] Switched to new setup")
def _eplb_reshuffle(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("perform_eplb_reshuffle",)
)
# Reshuffle changes per-rank token routing; the locked MoE workspace
# may now be too small. Rewarm covers both new and existing engines.
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("rewarm_workspace",)
)
assert self.new_dp_group is not None
if self.new_dp_group.rank() == 0:
logger.info("[Elastic EP] EPLB reshuffle completed")
def _eplb_reshuffle_before_scale_down(self):
def _commit_scale_down(self, removing: bool):
assert self.reconfig_request is not None and self.old_dp_group is not None
self.model_executor.collective_rpc(
self._collective_rpc(
"elastic_ep_execute",
args=(
"perform_scale_down_eplb_reshuffle",
"commit_scale_down",
self.reconfig_request.new_data_parallel_size,
removing,
),
)
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] EPLB reshuffle completed")
def _switch_and_remove(self):
self.model_executor.collective_rpc(
"elastic_ep_execute", args=("switch_and_remove",)
)
def _update_parallel_config(self):
assert self.reconfig_request is not None
reconfig_request = self.reconfig_request
+2 -1
View File
@@ -39,6 +39,7 @@ def create_standby_groups(
new_world_size_across_dp: int,
master_ip: str,
coord_store_port: int,
use_all2all: bool,
enable_eplb: bool = True,
backend: str | None = None,
) -> None:
@@ -86,7 +87,7 @@ def create_standby_groups(
)
standby_ep_ranks = [x.tolist() for x in standby_ep_ranks]
_STANDBY_EP = _init_stateless_group(
standby_ep_ranks, "ep", master_ip, backend, coord_store=coord_store
standby_ep_ranks, "ep", master_ip, backend, coord_store, use_all2all=use_all2all
)
if enable_eplb:
+13 -1
View File
@@ -414,6 +414,7 @@ class GroupCoordinator:
use_device_communicator: bool, # whether to use device communicator
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_all2all: bool = False,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -508,6 +509,7 @@ class GroupCoordinator:
device=self.device,
device_group=self.device_group,
unique_name=self.unique_name,
use_all2all=use_all2all,
)
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
@@ -1321,6 +1323,7 @@ def init_model_parallel_group(
use_message_queue_broadcaster: bool = False,
group_name: str | None = None,
use_device_communicator: bool = True,
use_all2all: bool = False,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
@@ -1329,6 +1332,7 @@ def init_model_parallel_group(
use_device_communicator=use_device_communicator,
use_message_queue_broadcaster=use_message_queue_broadcaster,
group_name=group_name,
use_all2all=use_all2all,
)
@@ -1339,6 +1343,7 @@ def _init_stateless_group(
backend: str,
coord_store: Store,
use_device_communicator: bool = True,
use_all2all: bool = False,
) -> "StatelessGroupCoordinator":
"""Create a StatelessGroupCoordinator with the given parameters."""
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
@@ -1354,6 +1359,7 @@ def _init_stateless_group(
coord_store=coord_store,
global_rank=world.rank,
global_world_size=world.world_size,
use_all2all=use_all2all,
)
@@ -1924,6 +1930,7 @@ def initialize_model_parallel(
.unbind(0)
)
group_ranks = [x.tolist() for x in group_ranks]
use_all2all = parallel_config.use_all2all
if enable_elastic_ep:
_EP = _init_stateless_group(
group_ranks,
@@ -1931,10 +1938,15 @@ def initialize_model_parallel(
parallel_config.data_parallel_master_ip,
backend,
coord_store=coord_store,
use_all2all=use_all2all,
)
else:
_EP = init_model_parallel_group(
group_ranks, get_world_group().local_rank, backend, group_name="ep"
group_ranks,
get_world_group().local_rank,
backend,
group_name="ep",
use_all2all=use_all2all,
)
# Create EPLB group with the same ranks as EP if EPLB is enabled.
@@ -79,6 +79,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
host: str = "127.0.0.1",
global_rank: int = 0,
global_world_size: int = 1,
use_all2all: bool = False,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -191,6 +192,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
global_ranks=self.ranks,
global_world_size=global_world_size,
tcp_store_group=self.tcp_store_group,
use_all2all=use_all2all,
)
self.mq_broadcaster = None
@@ -12,10 +12,7 @@ from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
)
from vllm.entrypoints.serve.elastic_ep.middleware import (
get_scaling_elastic_ep,
set_scaling_elastic_ep,
)
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -64,8 +61,6 @@ async def scale_elastic_ep(raw_request: Request):
status_code=400, detail="drain_timeout must be a positive integer"
)
# Set scaling flag to prevent new requests
set_scaling_elastic_ep(True)
client = engine_client(raw_request)
try:
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
@@ -83,8 +78,6 @@ async def scale_elastic_ep(raw_request: Request):
except Exception as e:
logger.error("Scale failed: %s", e)
raise HTTPException(status_code=500, detail="Scale failed") from e
finally:
set_scaling_elastic_ep(False)
@router.post("/is_scaling_elastic_ep")
+17 -14
View File
@@ -93,7 +93,7 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None:
)
def kernel_warmup(worker: "Worker"):
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
minimax_m3_msa_warmup,
)
@@ -118,6 +118,22 @@ def kernel_warmup(worker: "Worker"):
)
# Run next so input-prep kernels JIT against pristine runner state.
if worker.vllm_config.kernel_config.enable_jit_warmup:
fa4_cutedsl_warmup(worker)
sparse_mla_triton_warmup(worker)
if current_platform.has_device_capability(90):
_warmup_ll_bf16_router_gemm(worker.get_model())
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
# to the shared JIT warmup infrastructure.
# https://github.com/vllm-project/vllm/pull/47451
cutedsl_warmup()
if process_local_only:
return
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
@@ -143,9 +159,6 @@ def kernel_warmup(worker: "Worker"):
elif has_flashinfer() and current_platform.has_device_capability(90):
flashinfer_autotune(worker.model_runner)
if current_platform.has_device_capability(90):
_warmup_ll_bf16_router_gemm(worker.get_model())
# FlashInfer attention warmup
# Only warmup if the model has FlashInfer attention groups
# and is not a pooling model
@@ -178,16 +191,6 @@ def kernel_warmup(worker: "Worker"):
create_mixed_batch=True,
)
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
# to the shared JIT warmup infrastructure.
# https://github.com/vllm-project/vllm/pull/47451
cutedsl_warmup()
if worker.vllm_config.kernel_config.enable_jit_warmup:
fa4_cutedsl_warmup(worker)
sparse_mla_triton_warmup(worker)
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
-2
View File
@@ -35,8 +35,6 @@ FT_STATUS_CALL_ID = -2
class EEPNotificationType(enum.Enum):
NEW_CORE_ENGINES_INIT_READY = "NEW_CORE_ENGINES_INIT_READY"
NEW_CORE_ENGINES_WEIGHTS_INIT_READY = "NEW_CORE_ENGINES_WEIGHTS_INIT_READY"
RECONFIGURE_FINISHED = "RECONFIGURE_FINISHED"
SHUTDOWN_COMPLETE = "SHUTDOWN_COMPLETE"
+26 -19
View File
@@ -109,6 +109,7 @@ class AsyncLLM(EngineClient):
maybe_register_config_serialize_by_value()
self.vllm_config = vllm_config
self._elastic_ep_lock = asyncio.Lock()
self.model_config = vllm_config.model_config
self.observability_config = vllm_config.observability_config
@@ -998,17 +999,27 @@ class AsyncLLM(EngineClient):
"waiting for requests to drain."
)
async def _drain_requests_for_elastic_ep(self, drain_timeout: int) -> None:
try:
logger.info(
"VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, "
"waiting for requests to drain before scaling"
)
await self.wait_for_requests_to_drain(drain_timeout)
except BaseException:
set_scaling_elastic_ep(False)
raise
async def scale_elastic_ep(
self, new_data_parallel_size: int, drain_timeout: int = 300
):
"""
Scale up or down the data parallel size by adding or removing
engine cores.
Args:
new_data_parallel_size: The new number of data parallel workers
drain_timeout:
Maximum time to wait for requests to drain (seconds)
"""
"""Scale the elastic EP data parallel size."""
async with self._elastic_ep_lock:
await self._scale_elastic_ep(new_data_parallel_size, drain_timeout)
async def _scale_elastic_ep(
self, new_data_parallel_size: int, drain_timeout: int
) -> None:
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
if old_data_parallel_size == new_data_parallel_size:
logger.info(
@@ -1017,12 +1028,7 @@ class AsyncLLM(EngineClient):
)
return
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
logger.info(
"VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, "
"waiting for requests to drain before scaling"
)
await self.wait_for_requests_to_drain(drain_timeout)
await self.engine_core.prepare_elastic_ep(new_data_parallel_size)
# recreate stat loggers
if new_data_parallel_size > old_data_parallel_size and self.log_stats:
@@ -1042,11 +1048,12 @@ class AsyncLLM(EngineClient):
self.logger_manager.log_engine_initialized()
set_scaling_elastic_ep(True)
try:
await self.engine_core.scale_elastic_ep(new_data_parallel_size)
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
finally:
set_scaling_elastic_ep(False)
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
await self._drain_requests_for_elastic_ep(drain_timeout)
await self.engine_core.commit_elastic_ep()
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
set_scaling_elastic_ep(False)
async def handle_fault(
self, fault_tolerance_request: FaultToleranceRequest
+3
View File
@@ -376,6 +376,9 @@ class DPCoordinatorProc:
eng_index = outputs.engine_index
scheduler_stats = outputs.scheduler_stats
if scheduler_stats:
# Elastic EP stats may arrive while the engine list changes.
if eng_index >= len(self.engines):
continue
# 1. Updated request load stats - update our local
# state with these.
stats = self.engines[eng_index].request_counts
+36 -39
View File
@@ -327,8 +327,9 @@ class EngineCore:
vllm_config.validate_block_size()
# Initialize kv cache and warmup the execution
self.model_executor.initialize_from_config(kv_cache_configs)
if not envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
self.model_executor.compile_or_warm_up_model()
elapsed = time.time() - start
compile_time = vllm_config.compilation_config.compilation_time
@@ -993,9 +994,7 @@ class EngineCore:
raise NotImplementedError
def _eep_send_engine_core_notification(
self,
notification_type: EEPNotificationType,
vllm_config: VllmConfig | None = None,
self, notification_type: EEPNotificationType
):
raise NotImplementedError
@@ -1070,11 +1069,6 @@ class EngineCoreProc(EngineCore):
self.addresses = addresses
self.process_input_queue_block = True
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
self._eep_send_engine_core_notification(
EEPNotificationType.NEW_CORE_ENGINES_INIT_READY,
vllm_config=vllm_config,
)
self._init_data_parallel(vllm_config)
super().__init__(
@@ -2096,12 +2090,16 @@ class DPEngineCoreProc(EngineCoreProc):
self._maybe_publish_request_counts()
if self.eep_scaling_state is not None:
_ = self.eep_scaling_state.progress()
if self.eep_scaling_state.is_complete():
if self.eep_scaling_state.worker_type == "removing":
state = self.eep_scaling_state
if state.commit_requested or not state.is_ready_for_switch():
state.progress()
if state.is_complete():
if state.worker_type == "removing":
raise SystemExit
self.process_input_queue_block = True
self.eep_scaling_state = None
elif not state.commit_requested and state.is_ready_for_switch():
self.process_input_queue_block = True
executed = self._process_engine_step()
self._maybe_publish_request_counts()
@@ -2171,7 +2169,7 @@ class DPEngineCoreProc(EngineCoreProc):
def reinitialize_distributed(
self, reconfig_request: ReconfigureDistributedRequest
) -> None:
) -> str:
from copy import deepcopy
from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState
@@ -2203,7 +2201,10 @@ class DPEngineCoreProc(EngineCoreProc):
== ReconfigureRankType.SHUTDOWN_CURRENT_RANK
)
self.eep_scaling_state = ElasticEPScalingState(
if self.eep_scaling_state is not None:
raise RuntimeError("Elastic EP reconfiguration is already active")
state = ElasticEPScalingState(
model_executor=self.model_executor,
engine_core=self,
vllm_config=self.vllm_config,
@@ -2212,30 +2213,34 @@ class DPEngineCoreProc(EngineCoreProc):
scale_type="scale_down" if is_scale_down else "scale_up",
reconfig_request=reconfig_request,
)
self.eep_scaling_state = state
self.process_input_queue_block = False
logger.info(
"[Elastic EP] Received reconfiguration request and starting scaling up/down"
)
return state.ready_key
def commit_prepared_elastic_ep(self) -> None:
state = self.eep_scaling_state
if state is None or state.commit_requested or not state.is_ready_for_switch():
raise RuntimeError("No prepared Elastic EP reconfiguration is ready")
state.commit_requested = True
self.process_input_queue_block = False
logger.info("[Elastic EP] Committing prepared reconfiguration")
def _eep_send_engine_core_notification(
self,
notification_type: EEPNotificationType,
vllm_config: VllmConfig | None = None,
self, notification_type: EEPNotificationType
):
"""
Send notifications to EngineCoreClient, which can then forward
the notifications to other engine core processes. It is used for:
1) In scale up: new core engines to notify existing core engines
that they are ready;
2) In scale down: removing core engines to notify EngineCoreClient
1) In scale down: removing core engines to notify EngineCoreClient
so EngineCoreClient can release their ray placement groups;
3) Both scale up/down: to notify EngineCoreClient that existing
2) Both scale up/down: to notify EngineCoreClient that existing
core engines have already switched to the new parallel setup.
"""
if vllm_config is None:
dp_rank = self.vllm_config.parallel_config.data_parallel_rank
else:
dp_rank = vllm_config.parallel_config.data_parallel_rank
dp_rank = self.vllm_config.parallel_config.data_parallel_rank
notification_data = (notification_type.value, dp_rank)
outputs = EngineCoreOutputs(
utility_output=UtilityOutput(
@@ -2257,22 +2262,11 @@ class DPEngineCoreProc(EngineCoreProc):
):
socket.send_multipart(encoder.encode(outputs))
def eep_handle_engine_core_notification(
self, notification_type: str | EEPNotificationType
):
"""
Handle notification received from EngineCoreClient
(forwarded from new core engines).
"""
assert self.eep_scaling_state is not None
if isinstance(notification_type, str):
notification_type = EEPNotificationType(notification_type)
self.eep_scaling_state.handle_notification(notification_type)
def _eep_scale_up_before_kv_init(self):
from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState
self.eep_scaling_state = ElasticEPScalingState(
self.ignore_start_dp_wave = True
state = ElasticEPScalingState(
model_executor=self.model_executor,
engine_core=self,
vllm_config=self.vllm_config,
@@ -2281,7 +2275,10 @@ class DPEngineCoreProc(EngineCoreProc):
scale_type="scale_up",
reconfig_request=None,
)
self.eep_scaling_state.run_pre_kv_init_states()
if self.eep_scaling_state is not None:
raise RuntimeError("Elastic EP reconfiguration is already active")
self.eep_scaling_state = state
state.run_pre_kv_init_states()
self.process_input_queue_block = False
+174 -136
View File
@@ -225,7 +225,10 @@ class EngineCoreClient(ABC):
running state."""
raise NotImplementedError
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
async def commit_elastic_ep(self) -> None:
raise NotImplementedError
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
raise NotImplementedError
async def get_output_async(self) -> EngineCoreOutputs:
@@ -1490,6 +1493,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
)
assert len(self.core_engines) > 1
self._prepared_elastic_ep: tuple[int, int] | None = None
self.eng_start_index = (
len(self.core_engines) * self.client_index
@@ -1603,31 +1607,16 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
if len(cache.pending_notifications[notification_type]) >= abs(
cache.num_new_core_engines
):
if notification_type == EEPNotificationType.SHUTDOWN_COMPLETE:
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
assert cache.num_new_core_engines < 0
old_dp_size = len(cache.existing_core_engines)
new_dp_size = old_dp_size + cache.num_new_core_engines
self.resources.engine_manager.scale_down_elastic_ep(
old_dp_size, new_dp_size
)
else:
await asyncio.gather(
*[
self._call_utility_async(
"eep_handle_engine_core_notification",
notification_type,
engine=engine,
)
for engine in cache.existing_core_engines
]
)
cache.pending_notifications[notification_type] = set()
if notification_type in [
EEPNotificationType.SHUTDOWN_COMPLETE,
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY,
]:
self.eep_scaling_cache = None
engine_manager = self.resources.engine_manager
assert isinstance(engine_manager, CoreEngineActorManager)
assert cache.num_new_core_engines < 0
old_dp_size = len(cache.existing_core_engines)
new_dp_size = old_dp_size + cache.num_new_core_engines
engine_manager.scale_down_elastic_ep(old_dp_size, new_dp_size)
self.vllm_config.parallel_config.data_parallel_size_local = len(
engine_manager.local_engine_actors
)
self.eep_scaling_cache = None
async def abort_requests_async(self, request_ids: list[str]) -> None:
if not request_ids or self.resources.engine_dead:
@@ -1651,31 +1640,46 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
) -> None:
await self._send_input(EngineCoreRequestType.ABORT, request_ids, engine)
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
"""Scale elastic EP data parallel size"""
async def commit_elastic_ep(self) -> None:
"""Commit prepared elastic EP scaling."""
prepared = self._prepared_elastic_ep
if prepared is None:
raise RuntimeError("Elastic EP scaling has not been prepared")
new_data_parallel_size, num_redundant_experts = prepared
cur_data_parallel_size = len(self.core_engines)
assert new_data_parallel_size != cur_data_parallel_size, (
f"new_data_parallel_size {new_data_parallel_size} must be "
f"different from cur_data_parallel_size {cur_data_parallel_size}"
if new_data_parallel_size > cur_data_parallel_size:
await self._commit_scale_up_elastic_ep(new_data_parallel_size)
else:
await self._commit_scale_down_elastic_ep(new_data_parallel_size)
self.vllm_config.parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
self._prepared_elastic_ep = None
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
"""Prepare elastic EP scaling without routing requests to new engines."""
if (prepared := self._prepared_elastic_ep) is not None:
if prepared[0] == new_data_parallel_size:
return
raise RuntimeError("Elastic EP scaling is already prepared")
cur_data_parallel_size = len(self.core_engines)
assert self.vllm_config.parallel_config.data_parallel_backend == "ray", (
"Only ray DP backend supports scaling elastic EP"
)
scale_up = new_data_parallel_size > cur_data_parallel_size
if scale_up:
await self._scale_up_elastic_ep(
cur_data_parallel_size, new_data_parallel_size
)
parallel_config = self.vllm_config.parallel_config
num_experts = self.vllm_config.model_config.get_num_experts()
num_redundant_experts = (
num_experts + parallel_config.eplb_config.num_redundant_experts
) * new_data_parallel_size // cur_data_parallel_size - num_experts
if new_data_parallel_size < cur_data_parallel_size:
await self._prepare_scale_down_elastic_ep(new_data_parallel_size)
else:
await self._scale_down_elastic_ep(
cur_data_parallel_size, new_data_parallel_size
await self._prepare_scale_up_elastic_ep(
new_data_parallel_size, num_redundant_experts
)
self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts
async def _eep_wait_for_setup_switch_complete(self) -> None:
def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future:
"""
Wait for core engines to switch to the new setup.
@@ -1687,9 +1691,26 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
future = asyncio.get_running_loop().create_future()
self.utility_results[EEP_NOTIFICATION_CALL_ID] = future
self._ensure_output_queue_task()
await future
return future
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
def _wait_for_new_engine_ready(self, new_core_engines: list[bytes]) -> None:
new_engine_identities = set(new_core_engines)
sync_input_socket = zmq.Socket.shadow(self.input_socket)
while new_engine_identities:
if not sync_input_socket.poll(timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000):
raise TimeoutError(
f"Timed out waiting for new engine core processes to "
f"start. Waited "
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
f"timeout, set the environment variable: "
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
)
identity, payload = sync_input_socket.recv_multipart()
new_engine_identities.discard(identity)
self._apply_ready_response(payload)
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
from vllm.distributed.utils import create_tcp_store
from vllm.utils.network_utils import get_open_ports_list
@@ -1709,36 +1730,36 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
)
parallel_config._coord_store_port = store.port
self._coord_store = store
return ip, store.port
async def _scale_up_elastic_ep(
self, cur_data_parallel_size: int, new_data_parallel_size: int
) -> None:
"""Scale up the data parallel size by creating new engine cores
and reconfiguring existing ones."""
cur_data_parallel_size = len(self.core_engines)
self.eep_scaling_cache = ElasticScalingCache(
existing_core_engines=self.core_engines.copy(),
num_new_core_engines=new_data_parallel_size - cur_data_parallel_size,
pending_notifications=dict(),
def _make_reconfig_request(
self,
new_data_parallel_size: int,
rank_type: ReconfigureRankType = ReconfigureRankType.KEEP_CURRENT_RANK,
) -> ReconfigureDistributedRequest:
parallel_config = self.vllm_config.parallel_config
return ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=rank_type,
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=parallel_config.data_parallel_master_ip,
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
coord_store_port=parallel_config._coord_store_port,
)
parallel_config = self.vllm_config.parallel_config
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
async def _prepare_scale_up_elastic_ep(
self,
new_data_parallel_size: int,
num_redundant_experts: int,
) -> None:
"""Prepare scale up by creating new engine cores and reconfiguring
existing ones."""
self._setup_elastic_ep_reconfig_bootstrap()
# Phase 1: Send reconfig messages to existing engines
reconfig_futures = []
for engine in self.core_engines:
reconfig_request = ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=ip,
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
coord_store_port=coord_store_port,
)
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
coro = self._call_utility_async(
"reinitialize_distributed", reconfig_request, engine=engine
)
@@ -1746,51 +1767,54 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
# Phase 2: Create new engines
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
parallel_config.eplb_config.num_redundant_experts = 0
start_new_worker_future = asyncio.to_thread(
self.resources.engine_manager.scale_up_elastic_ep,
self.vllm_config,
new_data_parallel_size,
num_redundant_experts,
)
wait_future = self._eep_wait_for_setup_switch_complete()
# Phase 3: Wait for new engines to be created
# and reconfig messages to be received
await asyncio.gather(start_new_worker_future, *reconfig_futures)
ready_keys = [future.result() for future in reconfig_futures]
ready_keys.extend(
f"eep_ready/{rank}"
for rank in range(len(self.core_engines), new_data_parallel_size)
)
await asyncio.to_thread(self._coord_store.wait, ready_keys)
logger.info("[Elastic EP] Successfully started new engines")
# Create new CoreEngine objects for the new engines
new_engine_identities = set()
for i in range(cur_data_parallel_size, new_data_parallel_size):
new_engine = i.to_bytes(2, "little")
self.core_engines.append(new_engine)
# NOTE(yongji): we don't update lb_engines here,
# we let run_engine_stats_update_task to update it.
new_engine_identities.add(new_engine)
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
new_core_engines = [
rank.to_bytes(2, "little")
for rank in range(len(self.core_engines), new_data_parallel_size)
]
# Wait for ready messages from new engines on the input socket
sync_input_socket = zmq.Socket.shadow(self.input_socket)
while new_engine_identities:
if not sync_input_socket.poll(
timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000 # convert to ms
):
raise TimeoutError(
f"Timed out waiting for new engine core processes to "
f"start. Waited "
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
f"timeout, set the environment variable: "
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
)
identity, payload = sync_input_socket.recv_multipart()
new_engine_identities.discard(identity)
self._apply_ready_response(payload)
await self.pause_scheduler_async(mode="keep", clear_cache=False)
wait_future = self._eep_wait_for_setup_switch_complete()
finish_futures = [
asyncio.create_task(
self._call_utility_async("commit_prepared_elastic_ep", engine=engine)
)
for engine in self.core_engines
]
try:
await asyncio.gather(*finish_futures)
await wait_future
self._wait_for_new_engine_ready(new_core_engines)
except Exception:
wait_future.cancel()
raise
# NOTE(yongji): Before we schedule any requests on the new workers,
# we should wait for them to switch to the new setup.
await wait_future
self.core_engines.extend(new_core_engines)
# Update the parallel config
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
parallel_config = self.vllm_config.parallel_config
parallel_config.data_parallel_size = new_data_parallel_size
if isinstance(self.resources.engine_manager, CoreEngineActorManager):
parallel_config.data_parallel_size_local = len(
self.resources.engine_manager.local_engine_actors
)
# Notify coordinator about scale up through existing
# stats_update_task connection
self._ensure_stats_update_task()
@@ -1803,10 +1827,23 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
"[Elastic EP] Scale up completed, new data parallel size: %s",
new_data_parallel_size,
)
await self.resume_scheduler_async()
async def _scale_down_elastic_ep(
self, cur_data_parallel_size: int, new_data_parallel_size: int
) -> None:
async def _prepare_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
self._setup_elastic_ep_reconfig_bootstrap()
reconfig_futures = []
for engine in self.core_engines[:new_data_parallel_size]:
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
coro = self._call_utility_async(
"reinitialize_distributed", reconfig_request, engine=engine
)
reconfig_futures.append(asyncio.create_task(coro))
ready_keys = await asyncio.gather(*reconfig_futures)
await asyncio.to_thread(self._coord_store.wait, ready_keys)
async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
"""Scale down the data parallel size by shutting down and
reconfiguring existing engine cores."""
cur_data_parallel_size = len(self.core_engines)
@@ -1817,50 +1854,51 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
pending_notifications=dict(),
)
parallel_config = self.vllm_config.parallel_config
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
old_core_engines = self.core_engines
# NOTE(yongji): Immediately stop sending requests to the removing engines.
self.core_engines = old_core_engines[:new_data_parallel_size]
self.lb_engines = self.lb_engines[:new_data_parallel_size]
removed_dp_size = cur_data_parallel_size - new_data_parallel_size
pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size
pause_futures = [
self._call_utility_async("pause_scheduler", mode, False, engine=engine)
for mode, engine in zip(pause_modes, old_core_engines)
]
await asyncio.gather(*pause_futures)
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
self.resources.engine_manager.remove_run_refs_for_scale_down(removed_dp_size)
wait_future = self._eep_wait_for_setup_switch_complete()
reconfig_futures = []
for cur_dp_rank, engine in enumerate(self.core_engines):
reconfig_request = ReconfigureDistributedRequest(
new_data_parallel_size=new_data_parallel_size,
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
new_data_parallel_master_ip=ip,
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
coord_store_port=coord_store_port,
)
if cur_dp_rank >= new_data_parallel_size:
reconfig_request.new_data_parallel_rank = (
ReconfigureRankType.SHUTDOWN_CURRENT_RANK
for cur_dp_rank, engine in enumerate(old_core_engines):
if cur_dp_rank < new_data_parallel_size:
coro = self._call_utility_async(
"commit_prepared_elastic_ep", engine=engine
)
else:
reconfig_request = self._make_reconfig_request(
new_data_parallel_size,
ReconfigureRankType.SHUTDOWN_CURRENT_RANK,
)
coro = self._call_utility_async(
"reinitialize_distributed", reconfig_request, engine=engine
)
coro = self._call_utility_async(
"reinitialize_distributed", reconfig_request, engine=engine
)
reconfig_futures.append(asyncio.create_task(coro))
# NOTE(yongji): Immediately stop sending requests to the removing engines.
self.core_engines = self.core_engines[:new_data_parallel_size]
self.lb_engines = self.lb_engines[:new_data_parallel_size]
wait_future = self._eep_wait_for_setup_switch_complete()
try:
await asyncio.gather(*reconfig_futures)
await asyncio.gather(*reconfig_futures)
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
self._ensure_stats_update_task()
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_down_marker)
await wait_future
await self.resume_scheduler_async()
except Exception:
wait_future.cancel()
raise
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
self._ensure_stats_update_task()
scale_down_marker = msgspec.msgpack.encode(
("SCALE_ELASTIC_EP", new_data_parallel_size)
)
await self.first_req_send_socket.send(scale_down_marker)
# NOTE(yongji): Unlike scaling up,
# here we don't actually need to wait for the setup switch to complete.
# We may want to remove it in the future.
await wait_future
logger.info(
"[Elastic EP] Scale down completed, new data parallel size: %s",
new_data_parallel_size,
+8 -23
View File
@@ -821,7 +821,10 @@ class CoreEngineActorManager:
return placement_groups, local_dp_ranks
def scale_up_elastic_ep(
self, cur_vllm_config: VllmConfig, new_data_parallel_size: int
self,
cur_vllm_config: VllmConfig,
new_data_parallel_size: int,
num_redundant_experts: int,
) -> None:
import copy
@@ -864,6 +867,9 @@ class CoreEngineActorManager:
if new_data_parallel_size > 1:
_apply_dp_identity_suffix(dp_vllm_config, rank)
dp_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
dp_vllm_config.parallel_config.eplb_config.num_redundant_experts = (
num_redundant_experts
)
dp_vllm_config.parallel_config.placement_group = pg
# Check if this placement group is on the head node
@@ -906,39 +912,18 @@ class CoreEngineActorManager:
self.created_placement_groups.append(pg)
self.placement_group_is_local.append(local_client)
ray.get(
[
actor.wait_for_init.remote()
for actor in (
self.local_engine_actors[-new_local_engines:]
if new_local_engines > 0
else []
)
+ self.remote_engine_actors[
-(len(placement_groups) - new_local_engines) :
]
]
)
actors = (
self.local_engine_actors[-new_local_engines:]
if new_local_engines > 0
else []
) + self.remote_engine_actors[-(len(placement_groups) - new_local_engines) :]
ray.get([actor.wait_for_init.remote() for actor in actors])
for actor in actors:
ref = actor.run.remote()
self.run_refs.append(ref)
self.actor_run_ref_dict[actor] = ref
cur_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
# Update old_vllm_config with new data_parallel_size_local if any new
# local engines were added
if new_local_engines > 0:
cur_vllm_config.parallel_config.data_parallel_size_local += (
new_local_engines
)
def scale_down_elastic_ep(
self, cur_data_parallel_size: int, new_data_parallel_size: int
) -> None:
+4 -4
View File
@@ -116,11 +116,11 @@ class Executor(ABC):
raise NotImplementedError
def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None:
"""
Initialize the KV caches and begin the model execution loop of the
underlying workers.
"""
"""Initialize the KV caches on the underlying workers."""
self.collective_rpc("initialize_from_config", args=(kv_cache_configs,))
def compile_or_warm_up_model(self) -> None:
"""Compile/warm up the model and capture cudagraphs on workers."""
compilation_times: list[CompilationTimes] = self.collective_rpc(
"compile_or_warm_up_model"
)