From 35efdf6b34f1ef76c23c8980f6e8a3b73cab50f1 Mon Sep 17 00:00:00 2001 From: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:19:13 +0300 Subject: [PATCH] [Elastic EP] Async preparation (#47288) Signed-off-by: Itay Alroy --- .buildkite/test_areas/expert_parallelism.yaml | 1 + tests/distributed/test_elastic_ep.py | 166 +++++-- vllm/config/parallel.py | 9 + .../base_device_communicator.py | 16 +- .../device_communicators/cpu_communicator.py | 5 +- .../device_communicators/cuda_communicator.py | 2 + .../device_communicators/xpu_communicator.py | 5 +- .../distributed/elastic_ep/elastic_execute.py | 181 +++---- vllm/distributed/elastic_ep/elastic_state.py | 460 ++++++------------ vllm/distributed/elastic_ep/standby_state.py | 3 +- vllm/distributed/parallel_state.py | 14 +- vllm/distributed/stateless_coordinator.py | 2 + .../serve/elastic_ep/api_router.py | 9 +- vllm/model_executor/warmup/kernel_warmup.py | 31 +- vllm/v1/engine/__init__.py | 2 - vllm/v1/engine/async_llm.py | 45 +- vllm/v1/engine/coordinator.py | 3 + vllm/v1/engine/core.py | 75 ++- vllm/v1/engine/core_client.py | 310 ++++++------ vllm/v1/engine/utils.py | 31 +- vllm/v1/executor/abstract.py | 8 +- 21 files changed, 701 insertions(+), 677 deletions(-) diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index a3b46b5828..1d1609d46b 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -52,4 +52,5 @@ steps: - vllm/compilation/ - tests/distributed/ commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - pytest -v -s distributed/test_elastic_ep.py diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py index 4ce7497598..01c254c2d0 100644 --- a/tests/distributed/test_elastic_ep.py +++ b/tests/distributed/test_elastic_ep.py @@ -3,7 +3,9 @@ import os import subprocess +import threading import time +from concurrent.futures import ThreadPoolExecutor import pytest import requests @@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool: return False +def _traffic_loop( + server: RemoteOpenAIServer, + dp_rank: int | None, + ready: threading.Barrier, + stop: threading.Event, + finished: threading.Event, + is_probe: bool = False, +) -> list[tuple[float, float, int | None]]: + url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions") + payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4} + headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)} + request_payload = None if is_probe else payload + responses = [] + is_ready = False + while not stop.is_set(): + request_start = time.perf_counter() + try: + response = requests.post( + url, json=request_payload, headers=headers, timeout=120 + ) + status_code = response.status_code + except requests.exceptions.RequestException: + status_code = None + responses.append((request_start, time.perf_counter(), status_code)) + if status_code == 200: + if not is_ready: + ready.wait(timeout=120) + is_ready = True + if finished.is_set(): + return responses + time.sleep(0.05) + return responses + + +def _downtime(responses: list[tuple[float, float, int | None]]) -> float: + rejected = [end for _, end, status in responses if status == 503] + if not rejected: + return 0 + recovered = next( + end for _, end, status in responses if status == 200 and end > rejected[-1] + ) + return recovered - rejected[0] + + +def _scale_with_traffic( + server: RemoteOpenAIServer, + source_dp_size: int, + new_dp_size: int, + traffic_mode: str, +) -> None: + traffic_clients: list[int | None] = [] + if traffic_mode == "light": + traffic_clients = [0] + elif traffic_mode == "heavy": + traffic_clients = [None] * source_dp_size + clients = [(None, True)] + [(rank, False) for rank in traffic_clients] + ready = threading.Barrier(len(clients) + 1) + stop = threading.Event() + finished = threading.Event() + + with ThreadPoolExecutor(max_workers=len(clients)) as executor: + futures = [ + executor.submit( + _traffic_loop, server, rank, ready, stop, finished, is_probe + ) + for rank, is_probe in clients + ] + try: + ready.wait(timeout=120) + start_time = time.perf_counter() + assert _send_scale_command(server, new_dp_size) + scale_seconds = time.perf_counter() - start_time + finished.set() + probe_result, *results = [future.result(timeout=120) for future in futures] + finally: + stop.set() + + bad_statuses = { + status + for responses in [probe_result, *results] + for _, _, status in responses + if status not in (200, 503) + } + assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}" + probe_503 = [start for start, _, status in probe_result if status == 503] + assert probe_503, "Scaling probe did not observe commit" + assert not results or any( + status == 200 and start_time <= request_start and request_end < probe_503[0] + for responses in results + for request_start, request_end, status in responses + ), "No request completed successfully during preparation" + + print( + f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]" + f"[traffic={traffic_mode}] " + f"scale_seconds={scale_seconds:.3f} " + f"downtime_seconds={_downtime(probe_result):.3f}" + ) + + def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float: assert server.port is not None result = evaluate_gsm8k( @@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float: return accuracy -def _base_serve_args(use_async_eplb: bool = False) -> list[str]: +def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]: args = [ "--trust-remote-code", "--tensor-parallel-size", @@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]: "--eplb-config.num_redundant_experts", "0", "--eplb-config.use_async", - "true" if use_async_eplb else "false", + "true", "--eplb-config.step_interval", - "10", + "300", "--eplb-config.window_size", "5", "--data-parallel-backend", "ray", "--data-parallel-size", - "2", + str(dp_size), "--api-server-count", "1", + "--disable-access-log-for-endpoints", + "/is_scaling_elastic_ep", ] leader_address = os.environ.get("LEADER_ADDRESS") if leader_address: args.extend(["--data-parallel-address", leader_address]) + if enforce_eager: + args.append("--enforce-eager") return args @pytest.mark.parametrize( - "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"] + ("enforce_eager", "traffic_mode"), + [ + pytest.param(True, "none", id="enforce_eager_none"), + pytest.param(True, "light", id="enforce_eager_light"), + pytest.param(True, "heavy", id="enforce_eager_heavy"), + pytest.param(False, "heavy", id="cuda_graphs_heavy"), + ], ) @multi_gpu_test(num_gpus=4) -def test_elastic_ep_scaling(use_async_eplb: bool): - if use_async_eplb: - from vllm.distributed.eplb.eplb_communicator import has_nixl +def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str): + from vllm.distributed.eplb.eplb_communicator import has_nixl - if not has_nixl(): - pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") + if not has_nixl(): + pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") - vllm_serve_args = _base_serve_args(use_async_eplb) + initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2")) + target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4")) + assert target_dp_size > initial_dp_size + vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager) with RemoteOpenAIServer( MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200 ) as server: - initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)") - - assert _send_scale_command(server, 4) - time.sleep(10) - scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)") + initial_accuracy = _run_gsm8k_eval(server, "Initial") + _scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode) + scale_up_accuracy = _run_gsm8k_eval(server, "After scale up") assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, ( f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than " f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}" ) - assert _send_scale_command(server, 2) - time.sleep(5) - scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)") - + _scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode) + scale_down_accuracy = _run_gsm8k_eval(server, "After scale down") assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, ( f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than " f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}" @@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool): print(f" Tolerance: {ACCURACY_TOL:.3f}") -@pytest.mark.parametrize( - "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"] -) @multi_gpu_test(num_gpus=4) -def test_elastic_ep_scaling_uneven(use_async_eplb: bool): +def test_elastic_ep_scaling_uneven(): """Test scale up with uneven worker distribution. This tests the case where num_new_workers % old_dp_size != 0, specifically 2 -> 3 where remainder = 1 % 2 = 1. This exercises the remainder handling in sender-receiver pairing. """ - if use_async_eplb: - from vllm.distributed.eplb.eplb_communicator import has_nixl + from vllm.distributed.eplb.eplb_communicator import has_nixl - if not has_nixl(): - pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") + if not has_nixl(): + pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") - vllm_serve_args = _base_serve_args(use_async_eplb) + vllm_serve_args = _base_serve_args() with RemoteOpenAIServer( MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200 @@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool): # Scale 2 -> 3: This has remainder = 1 % 2 = 1 # Tests uneven sender-receiver pairing assert _send_scale_command(server, 3) - time.sleep(10) scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)") assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, ( @@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool): # Scale back down to 2 assert _send_scale_command(server, 2) - time.sleep(5) scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)") assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, ( diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 949eb298a1..5ebc410f3c 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -686,6 +686,14 @@ class ParallelConfig: and self.data_parallel_size > 1 ) + @property + def use_all2all(self) -> bool: + return ( + self.data_parallel_size > 1 + or self.use_sequence_parallel_moe + or (self.enable_expert_parallel and self.prefill_context_parallel_size > 1) + ) + @property def use_batched_dp_moe(self) -> bool: return ( @@ -786,6 +794,7 @@ class ParallelConfig: "data_parallel_master_ip", "data_parallel_master_port", "_data_parallel_master_port_list", + "_coord_store_port", "data_parallel_rpc_port", "rank", "master_addr", diff --git a/vllm/distributed/device_communicators/base_device_communicator.py b/vllm/distributed/device_communicators/base_device_communicator.py index 45438a5469..73fd1331f5 100644 --- a/vllm/distributed/device_communicators/base_device_communicator.py +++ b/vllm/distributed/device_communicators/base_device_communicator.py @@ -175,6 +175,7 @@ class DeviceCommunicatorBase: unique_name: str = "", global_ranks: list[int] | None = None, global_world_size: int | None = None, + use_all2all: bool = False, ): self.device = device or torch.device("cpu") self.cpu_group = cpu_group @@ -204,26 +205,15 @@ class DeviceCommunicatorBase: self.global_world_size = dist.get_world_size() self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank) - use_ep = False all2all_backend = None from vllm.config import get_current_vllm_config_or_none config = get_current_vllm_config_or_none() if config is not None: - # initialize the all2all manager for DP or sequence-parallel EP. - parallel_config = config.parallel_config - use_ep = ( - parallel_config.data_parallel_size > 1 - or parallel_config.use_sequence_parallel_moe - or ( - parallel_config.enable_expert_parallel - and parallel_config.prefill_context_parallel_size > 1 - ) - ) - all2all_backend = parallel_config.all2all_backend + all2all_backend = config.parallel_config.all2all_backend self.is_ep_communicator = unique_name.split(":")[0] == "ep" - self.use_all2all = self.is_ep_communicator and use_ep + self.use_all2all = self.is_ep_communicator and use_all2all self.all2all_backend = all2all_backend self.all2all_manager: All2AllManagerBase | None = None diff --git a/vllm/distributed/device_communicators/cpu_communicator.py b/vllm/distributed/device_communicators/cpu_communicator.py index 9ec4b72f80..8ea12d9255 100644 --- a/vllm/distributed/device_communicators/cpu_communicator.py +++ b/vllm/distributed/device_communicators/cpu_communicator.py @@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase): device: torch.device | None = None, device_group: ProcessGroup | None = None, unique_name: str = "", + use_all2all: bool = False, ): - super().__init__(cpu_group, device, device_group, unique_name) + super().__init__( + cpu_group, device, device_group, unique_name, use_all2all=use_all2all + ) self.dist_module = torch.distributed if ( diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index fccc6ba60c..23e37ca830 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase): global_ranks: list[int] | None = None, global_world_size: int | None = None, tcp_store_group: StatelessProcessGroup | None = None, + use_all2all: bool = False, ): super().__init__( cpu_group, @@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase): unique_name, global_ranks, global_world_size, + use_all2all=use_all2all, ) if "tp" not in unique_name: # custom allreduce or torch symm mem can be used only by tp diff --git a/vllm/distributed/device_communicators/xpu_communicator.py b/vllm/distributed/device_communicators/xpu_communicator.py index 1b6ce9e8aa..7ca132824e 100644 --- a/vllm/distributed/device_communicators/xpu_communicator.py +++ b/vllm/distributed/device_communicators/xpu_communicator.py @@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase): device: torch.device | None = None, device_group: ProcessGroup | None = None, unique_name: str = "", + use_all2all: bool = False, ): - super().__init__(cpu_group, device, device_group, unique_name) + super().__init__( + cpu_group, device, device_group, unique_name, use_all2all=use_all2all + ) self.ca_comm: None = None if self.use_all2all: if self.all2all_backend in ("naive", "allgather_reducescatter"): diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index b0c3740f57..cea7fcb2f0 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import copy import gc import weakref from collections.abc import Iterable, Sequence +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import replace from typing import TYPE_CHECKING @@ -43,6 +43,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig from vllm.model_executor.layers.fused_moe.eep_reconfigure import ( make_eep_staged_quant_method, ) +from vllm.model_executor.warmup.kernel_warmup import kernel_warmup from vllm.utils import is_moe_layer from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper @@ -145,6 +146,10 @@ class ElasticEPScalingExecutor: self.worker_ref = weakref.ref(worker) self.reconfig_request = None self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {} + self._async_executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="ElasticEPAsync" + ) + self._async_future: Future[None] | None = None @property def worker(self): @@ -159,59 +164,68 @@ class ElasticEPScalingExecutor: raise ValueError(f"Unknown execute method: {execute_method}") return method(*args, **kwargs) - def _set_eplb_suppressed(self, suppressed: bool) -> None: - self.worker.model_runner.eep_eplb_suppressed = suppressed - ep_group = get_standby_ep_group() or get_ep_group() - if ep_group.rank == 0: - logger.info( - "[Elastic EP] EPLB %s elastic scaling transition", - "disabled during" if suppressed else "re-enabled after", - ) + def start_async(self, execute_method: str, *args, **kwargs) -> str: + if self._async_future is not None: + raise RuntimeError("Another Elastic EP async method is active") + if args and isinstance(args[0], ReconfigureDistributedRequest): + self.reconfig_request = args[0] + dp_rank = self.worker.vllm_config.parallel_config.data_parallel_rank + done_key = f"eep_async/{execute_method}/{dp_rank}/{self.worker.rank}" + self._async_future = self._async_executor.submit( + self._run_async, execute_method, *args, **kwargs + ) + self._async_future.add_done_callback(lambda _: self._mark_async_done(done_key)) + return done_key + + def _run_async(self, execute_method: str, *args, **kwargs) -> None: + from vllm.platforms import current_platform + + self.worker.vllm_config.enable_trace_function_call_for_thread() + assert hasattr(self.worker, "device") + current_platform.set_device(self.worker.device) + with set_current_vllm_config(self.worker.vllm_config): + self.execute(execute_method, *args, **kwargs) + + def _mark_async_done(self, done_key: str) -> None: + from vllm.distributed.utils import get_cached_tcp_store_client + + assert self.reconfig_request is not None + get_cached_tcp_store_client( + self.reconfig_request.new_data_parallel_master_ip, + self.reconfig_request.coord_store_port, + ).set(done_key, b"1") + + def clear_async(self) -> None: + future = self._async_future + if future is None: + raise RuntimeError("No Elastic EP async method is active") + if not future.done(): + raise RuntimeError("Elastic EP async method is not done") + self._async_future = None + future.result() def load_model(self) -> None: - ( - expanded_physical_to_logical, - num_logical_experts, - old_num_physical_experts, - ) = self.receive_expert_mapping() - num_physical_experts = expanded_physical_to_logical.shape[1] - self.worker.parallel_config.eplb_config.num_redundant_experts = ( - num_physical_experts - num_logical_experts - ) self.worker.load_model(load_dummy_weights=True) - self.worker.model_runner.setup_eplb_from_mapping( - expanded_physical_to_logical, old_num_physical_experts - ) - self._set_eplb_suppressed(True) def create_standby_groups( - self, reconfig_request: ReconfigureDistributedRequest + self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool ) -> None: self.reconfig_request = reconfig_request new_dp_size = reconfig_request.new_data_parallel_size old_dp_size = get_dp_group().world_size - world_size = self.worker.vllm_config.parallel_config.world_size + parallel_config = self.worker.vllm_config.parallel_config + world_size = parallel_config.world_size new_world_size_across_dp = world_size * new_dp_size - updated_config = copy.copy(self.worker.vllm_config) - updated_config.parallel_config = copy.deepcopy( - self.worker.vllm_config.parallel_config + create_standby_groups( + new_dp_size=new_dp_size, + new_world_size_across_dp=new_world_size_across_dp, + master_ip=reconfig_request.new_data_parallel_master_ip, + coord_store_port=reconfig_request.coord_store_port, + use_all2all=use_all2all, + enable_eplb=parallel_config.enable_eplb, ) - updated_config.parallel_config.data_parallel_size = new_dp_size - with set_current_vllm_config(updated_config): - create_standby_groups( - new_dp_size=new_dp_size, - new_world_size_across_dp=new_world_size_across_dp, - master_ip=reconfig_request.new_data_parallel_master_ip, - coord_store_port=reconfig_request.coord_store_port, - enable_eplb=updated_config.parallel_config.enable_eplb, - ) - if new_dp_size > old_dp_size: - self._set_eplb_suppressed(True) - eplb_state = self.worker.model_runner.eplb_state - if eplb_state is not None: - eplb_state.drain_async() - elif new_dp_size < old_dp_size: - self._stage_standby_moe_quant_methods() + if new_dp_size < old_dp_size: + self.stage_standby_moe_quant_methods() def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None: standby_dp_group = get_standby_dp_group() @@ -265,6 +279,7 @@ class ElasticEPScalingExecutor: model_config = self.worker.model_runner.model_config eplb_state = self.worker.model_runner.eplb_state assert eplb_state is not None + eplb_state.drain_async() eplb_model_state = eplb_state.model_states[model_config.compute_hash()] physical_to_logical = eplb_model_state.physical_to_logical_map num_physical_experts = physical_to_logical.shape[1] @@ -278,10 +293,6 @@ class ElasticEPScalingExecutor: src_rank=0, device=self.worker.device, ) - # New workers enter load_model after receiving the expert mapping. - # Stage replacement MoE kernels before returning to the state machine - # so existing ranks can participate in collective EP comm creation. - self._stage_standby_moe_quant_methods() def _make_eep_moe_config(self, module, dp_group, ep_group): parallel_config = self.worker.vllm_config.parallel_config @@ -300,7 +311,7 @@ class ElasticEPScalingExecutor: moe_parallel_config=moe_parallel_config, ) - def _stage_standby_moe_quant_methods(self) -> None: + def stage_standby_moe_quant_methods(self) -> None: standby_dp_group = get_standby_dp_group() standby_ep_group = get_standby_ep_group() model = self.worker.model_runner.get_model() @@ -500,26 +511,6 @@ class ElasticEPScalingExecutor: compilation_counter.stock_torch_compile_count += 1 self.worker.model_runner.model.compile(fullgraph=True, backend=backend) - multi_block_table = self.worker.model_runner.input_batch.block_table - saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = [] - for bt in multi_block_table.block_tables: - saved_block_tables.append( - (bt.block_table.gpu.clone(), bt.block_table.cpu.clone()) - ) - multi_block_table.clear() - - unlock_workspace() - self.worker.compile_or_warm_up_model() - lock_workspace() - - for bt, (saved_gpu, saved_cpu) in zip( - multi_block_table.block_tables, saved_block_tables - ): - bt.block_table.gpu.copy_(saved_gpu) - bt.block_table.cpu.copy_(saved_cpu) - if new_dp_size < old_dp_size: - self._set_eplb_suppressed(False) - def _perform_eplb_reshuffle( self, rank_mapping: dict[int, int] | None = None ) -> None: @@ -553,12 +544,25 @@ class ElasticEPScalingExecutor: if get_ep_group().rank == 0: logger.info("[Elastic EP] Expert resharding completed") - def perform_eplb_reshuffle(self) -> None: + def commit_scale_up(self, is_existing_worker: bool) -> None: + if is_existing_worker: + self.broadcast_expert_mapping() + self.switch_and_prepare() + else: + mapping, _, num_valid_experts = self.receive_expert_mapping() + self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts) self._perform_eplb_reshuffle() - self._set_eplb_suppressed(False) + self.warm_and_capture() + + def commit_scale_down(self, new_dp_size: int, removing: bool) -> None: + self.perform_scale_down_eplb_reshuffle(new_dp_size) + if removing: + self.switch_and_remove() + else: + self.switch_and_prepare() + self.warm_and_capture() def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None: - self._set_eplb_suppressed(True) eplb_state = self.worker.model_runner.eplb_state if eplb_state is not None: eplb_state.drain_async() @@ -599,12 +603,17 @@ class ElasticEPScalingExecutor: ) model = self.worker.model_runner.get_model() + expert_weights = [ + module.get_expert_weights() + for module in model.modules() + if is_moe_layer(module) + ] batch_transfer_weights( model=model, is_sender=False, peer_rank=sender_rank, dp_group=dp_group, - expert_weights=model.expert_weights, + expert_weights=expert_weights, ) torch.accelerator.synchronize() @@ -643,14 +652,17 @@ class ElasticEPScalingExecutor: with set_current_vllm_config(self.worker.vllm_config): prepare_communication_buffer_for_model(self.worker.model_runner.get_model()) - def rewarm_workspace(self) -> None: + def warmup_local_kernels(self) -> None: + with set_current_vllm_config(self.worker.vllm_config): + kernel_warmup(self.worker, process_local_only=True) + + def warm_and_capture(self) -> None: # Must run on every DP sibling in lockstep: _dummy_run calls # coordinate_batch_across_dp whenever data_parallel_size > 1 # (gpu_model_runner.py:3663), which deadlocks if any rank skips it. - # Save and clear block tables so profile_run/compile_or_warm_up_model - # don't write dummy slot mappings into real KV-cache blocks (mirrors - # switch_and_prepare's pattern). + # Save and clear block tables so the dummy MoE forward doesn't + # write dummy slot mappings into real KV-cache blocks. multi_block_table = self.worker.model_runner.input_batch.block_table saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = [] for bt in multi_block_table.block_tables: @@ -660,19 +672,16 @@ class ElasticEPScalingExecutor: multi_block_table.clear() # _ensure_workspace_size allocates a fresh tensor on grow, leaving - # captured CUDA graphs with stale data pointers; drop graphs before - # re-warm so captures realign with the resized buffer. + # any captured CUDA graph with a stale data pointer; drop graphs + # before re-warm so captures realign with the resized buffer. self._release_cuda_graphs() unlock_workspace() - # Grow the MoE workspace at max_num_tokens. - # compile_or_warm_up_model alone only exercises cudagraph-capture - # sizes (≤64 tokens for this test) and leaves the workspace at - # ~10-14 MB; the post-all-to-all per-rank token count under real - # post-reshuffle routing needs hundreds of MB. Use _dummy_run - # directly (rather than profile_run) with skip_eplb=True so dummy - # routing doesn't pollute the just-rebalanced EPLB stats — same - # convention compile_or_warm_up_model itself uses. + # Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model + # alone only exercises cudagraph-capture sizes and can leave the + # workspace too small for post-reshuffle routing. Use _dummy_run + # directly with skip_eplb=True so dummy routing doesn't pollute the + # just-rebalanced EPLB stats. runner = self.worker.model_runner runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True) self.worker.compile_or_warm_up_model() diff --git a/vllm/distributed/elastic_ep/elastic_state.py b/vllm/distributed/elastic_ep/elastic_state.py index 256efe46a4..f33fd90e86 100644 --- a/vllm/distributed/elastic_ep/elastic_state.py +++ b/vllm/distributed/elastic_ep/elastic_state.py @@ -1,18 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import enum -import time import weakref -from datetime import timedelta -from typing import TYPE_CHECKING, Literal, TypeAlias +from concurrent.futures import Future, ThreadPoolExecutor +from typing import TYPE_CHECKING, Any, Literal, TypeAlias import torch.distributed from vllm.config import ParallelConfig from vllm.distributed import ( - sched_yield, stateless_destroy_torch_distributed_process_group, ) +from vllm.distributed.utils import get_cached_tcp_store_client from vllm.logger import init_logger from vllm.v1.engine import ( EEPNotificationType, @@ -31,35 +30,29 @@ WorkerType = Literal["existing", "new", "removing"] class ScaleUpExistingEngineState(enum.IntEnum): - WAIT_NEW_CORE_ENGINES_INIT = 0 - CREATE_STANDBY_GROUPS = 1 - TRANSFER_EXPERT_MAPPING = 2 - WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT = 3 - TRANSFER_WEIGHTS = 4 - SYNC_KV_CACHE_MEMORY_SIZE = 5 - SWITCH_AND_PREPARE = 6 - EPLB_RESHUFFLE = 7 - COMPLETE = 8 + CREATE_STANDBY_GROUPS = 0 + STAGE_QUANT_METHODS = 1 + TRANSFER_WEIGHTS = 2 + SYNC_KV_CACHE_MEMORY_SIZE = 3 + COMMIT_SCALE_UP = 4 # Blocks forward passes. + COMPLETE = 5 class ScaleUpNewEngineState(enum.IntEnum): PRE_KV_INIT = 0 PREPARE = 1 - EPLB_RESHUFFLE = 2 - COMPLETE = 3 + COMPLETE = 2 class ScaleDownRemainingEngineState(enum.IntEnum): PREPARE = 0 - EPLB_RESHUFFLE = 1 - SWITCH_AND_PREPARE = 2 - COMPLETE = 3 + COMMIT_SCALE_DOWN = 1 # Blocks forward passes. + COMPLETE = 2 class ScaleDownRemovingEngineState(enum.IntEnum): PREPARE = 0 - EPLB_RESHUFFLE = 1 - COMPLETE = 2 + COMPLETE = 1 EngineState: TypeAlias = ( @@ -70,15 +63,6 @@ EngineState: TypeAlias = ( ) -class _BarrierTimeoutError(RuntimeError): - """ - Exception raised for timeout - in the first stage of our two-staged - TCPStore based barrier to synchronize the - execution of all engines in the DP group. - """ - - class ElasticEPScalingState: def __init__( self, @@ -94,20 +78,24 @@ class ElasticEPScalingState: self.engine_core_ref = weakref.ref(engine_core) self.vllm_config = vllm_config self.old_dp_group = self.engine_core.dp_group if worker_type != "new" else None - self.old_dp_store = self.engine_core.dp_store if worker_type != "new" else None self.new_parallel_config: ParallelConfig = new_parallel_config self.new_dp_group = self.engine_core.dp_group if worker_type == "new" else None self.new_dp_store = self.engine_core.dp_store if worker_type == "new" else None self.worker_type = worker_type self.scale_type = scale_type self.reconfig_request = reconfig_request - + self.commit_requested = False + self._prepare_executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="ElasticEPPrepare" + ) + self._prepare_future: Future[Any] | None = None + self._new_dp_sync: tuple[object, Any] | None = None self.state: EngineState if scale_type == "scale_up": self.state = ( ScaleUpNewEngineState.PRE_KV_INIT if worker_type == "new" - else ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT + else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS ) else: self.state = ( @@ -130,6 +118,31 @@ class ElasticEPScalingState: raise RuntimeError("Engine core has been garbage collected") return engine_core + def _collective_rpc(self, *args, **kwargs): + return self.model_executor.collective_rpc(*args, **kwargs) + + def _execute_async(self, execute_method: str, *args) -> bool: + if self._prepare_future is None: + done_keys = self._collective_rpc( + "elastic_ep_execute", + args=("start_async", execute_method, *args), + ) + assert self.reconfig_request is not None + coord_store = get_cached_tcp_store_client( + self.reconfig_request.new_data_parallel_master_ip, + self.reconfig_request.coord_store_port, + ) + self._prepare_future = self._prepare_executor.submit( + coord_store.wait, done_keys + ) + if not self._prepare_future.done(): + return False + + self._prepare_future.result() + self._collective_rpc("elastic_ep_execute", args=("clear_async",)) + self._prepare_future = None + return True + def progress(self) -> bool: if self.scale_type == "scale_up": return ( @@ -149,157 +162,43 @@ class ElasticEPScalingState: assert self.progress() assert self.state == ScaleUpNewEngineState.PREPARE - def _execute_tcp_store_barrier( - self, dp_store, group_rank, group_size, barrier_id, timeout=None - ): - arrival_key = f"arrival_{barrier_id}_{group_rank}" - dp_store.set(arrival_key, b"1") - - start_time = time.time() - processes_arrived: set[int] = set() - - while len(processes_arrived) < group_size: - if ( - timeout is not None - and time.time() - start_time > timeout.total_seconds() - ): - raise _BarrierTimeoutError( - f"Barrier timed out after {timeout.total_seconds()} seconds" - ) - - for i in range(group_size): - if i in processes_arrived: - continue - - key = f"arrival_{barrier_id}_{i}" - present = dp_store.check([key]) - if present: - processes_arrived.add(i) - - if len(processes_arrived) < group_size: - sched_yield() - - def _staged_barrier(self, use_new_group: bool, barrier_name: str) -> bool: - """ - Execute a two-staged barrier to synchronize all engines in the DP group. - - Some DP EngineCores may receive the reconfiguration notifications - later than others, and already proceed to engine step (model forward) - in the busy loop. - In this case, EngineCores that already proceed to reconfiguration - should skip reconfiguration and execute model forward for one more - step, so in the next step, all EngineCores will be synchronized. - We use a two-staged barrier to achieve this. The first time each - EngineCore executes the barrier, if a timeout is reached before the - barrier completes, that means some EngineCores have already entered - engine step. The EngineCores that timed out will then proceed to - engine step, and will synchronize with the other EngineCores in the - next step with a barrier without timeout. - """ - dp_group = self.new_dp_group if use_new_group else self.old_dp_group - dp_store = self.new_dp_store if use_new_group else self.old_dp_store - assert dp_group is not None and dp_store is not None - - group_rank = dp_group.rank() - group_size = dp_group.size() - barrier_id = f"eep_barrier_{barrier_name}" - sync_key = f"{barrier_id}_sync" - - # TODO(yongji): figure out appropriate timeout for the barrier - timeout = None if dp_store.check([sync_key]) else timedelta(seconds=5) - - try: - self._execute_tcp_store_barrier( - dp_store, group_rank, group_size, barrier_id, timeout=timeout - ) - torch.distributed.barrier(dp_group) - if group_rank == 0: - dp_store.delete_key(sync_key) - for i in range(group_size): - dp_store.delete_key(f"arrival_{barrier_id}_{i}") - return True - except _BarrierTimeoutError as e: - if timeout is None: - raise RuntimeError("Unexpected timeout encountered") from e - dp_store.compare_set(sync_key, "", b"1") - return False - def _progress_existing_engine(self) -> bool: state = self.state - assert self.old_dp_group is not None and self.old_dp_store is not None + assert self.old_dp_group is not None - if state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT: - return False - - elif state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS: - # NOTE(yongji): wait for all existing workers to receive the request - if ( - int(self.old_dp_store.get("eep_barrier_engine_count")) - < self.old_dp_group.size() - ): + if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS: + if not self._create_standby_groups(): return False - if not self._staged_barrier( - use_new_group=False, barrier_name="create_standby_groups" - ): - return False - if self.old_dp_group.rank() == 0: - self.old_dp_store.delete_key("eep_barrier_engine_count") - self._create_standby_groups() - self.state = ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING + self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS return True - elif state == ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING: - self._transfer_expert_mapping() - self.state = ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT + elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS: + if not self._execute_async("stage_standby_moe_quant_methods"): + return False + self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS return True - elif state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT: - return False - elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS: - if ( - int(self.old_dp_store.get("eep_barrier_engine_count")) - < self.old_dp_group.size() - ): + if not self._transfer_weights(): return False - if not self._staged_barrier( - use_new_group=False, barrier_name="transfer_weights" - ): - return False - if self.old_dp_group.rank() == 0: - self.old_dp_store.delete_key("eep_barrier_engine_count") - self._transfer_weights() self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE return True elif state == ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE: - self._sync_kv_cache_memory_size() - self.state = ScaleUpExistingEngineState.SWITCH_AND_PREPARE + if not self._sync_kv_cache_memory_size(): + return False + self.state = ScaleUpExistingEngineState.COMMIT_SCALE_UP + self._mark_ready_for_switch() return True - elif state == ScaleUpExistingEngineState.SWITCH_AND_PREPARE: - self._switch_and_prepare() - self.state = ScaleUpExistingEngineState.EPLB_RESHUFFLE - assert self.new_dp_store is not None - self.new_dp_store.add("eep_barrier_engine_count", 1) - return True - - elif state == ScaleUpExistingEngineState.EPLB_RESHUFFLE: - assert self.new_dp_group is not None and self.new_dp_store is not None - if ( - int(self.new_dp_store.get("eep_barrier_engine_count")) - < self.new_dp_group.size() - ): + elif state == ScaleUpExistingEngineState.COMMIT_SCALE_UP: + if not self.commit_requested: return False - if not self._staged_barrier( - use_new_group=True, barrier_name="eplb_reshuffle" - ): - return False - if self.new_dp_group.rank() == 0: - self.new_dp_store.delete_key("eep_barrier_engine_count") - self._eplb_reshuffle() + self._commit_new_dp_group() + self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", True)) self.state = ScaleUpExistingEngineState.COMPLETE self._update_parallel_config() + self._send_reconfigure_finished() return True else: @@ -311,22 +210,17 @@ class ElasticEPScalingState: assert self.new_dp_group is not None and self.new_dp_store is not None if state == ScaleUpNewEngineState.PRE_KV_INIT: - self.engine_core._eep_send_engine_core_notification( - EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY - ) - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("receive_weights",) - ) + self._collective_rpc("elastic_ep_execute", args=("receive_weights",)) self.engine_core.available_gpu_memory_for_kv_cache = ( ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1) ) - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("prepare_new_worker",) - ) + self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",)) self.state = ScaleUpNewEngineState.PREPARE return True elif state == ScaleUpNewEngineState.PREPARE: + self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",)) + self._mark_ready_for_switch() tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu") torch.distributed.all_reduce( tensor, @@ -337,22 +231,7 @@ class ElasticEPScalingState: self.engine_core.engines_running = bool(data[0]) self.engine_core.current_wave = int(data[1]) self.engine_core.step_counter = int(data[2]) - self.state = ScaleUpNewEngineState.EPLB_RESHUFFLE - self.new_dp_store.add("eep_barrier_engine_count", 1) - return True - - elif state == ScaleUpNewEngineState.EPLB_RESHUFFLE: - if ( - int(self.new_dp_store.get("eep_barrier_engine_count")) - < self.new_dp_group.size() - ): - return False - if not self._staged_barrier( - use_new_group=True, barrier_name="eplb_reshuffle" - ): - return False - assert self.new_dp_group.rank() > 0 - self._eplb_reshuffle() + self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", False)) self.state = ScaleUpNewEngineState.COMPLETE return True @@ -362,38 +241,23 @@ class ElasticEPScalingState: def _progress_remaining_engine(self) -> bool: state = self.state - assert self.old_dp_group is not None and self.old_dp_store is not None + assert self.old_dp_group is not None if state == ScaleDownRemainingEngineState.PREPARE: - self.state = ScaleDownRemainingEngineState.EPLB_RESHUFFLE - self.old_dp_store.add("eep_barrier_engine_count", 1) - return True + if self._create_standby_groups(): + self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN + self._mark_ready_for_switch() + return True + return False - elif state == ScaleDownRemainingEngineState.EPLB_RESHUFFLE: - if ( - int(self.old_dp_store.get("eep_barrier_engine_count")) - < self.old_dp_group.size() - ): + elif state == ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN: + if not self.commit_requested: return False - if not self._staged_barrier( - use_new_group=False, barrier_name="eplb_reshuffle" - ): - return False - if self.old_dp_group.rank() == 0: - self.old_dp_store.delete_key("eep_barrier_engine_count") - self._eplb_reshuffle_before_scale_down() - self.state = ScaleDownRemainingEngineState.SWITCH_AND_PREPARE - # NOTE(yongji): currently, after EPLB reshuffle - # that redistributes experts to remaining workers, workers - # to be removed will immediately initiate shutdown; - # existing workers can no longer execute forward steps using - # the old setup. In the future, we may keep - # the removing workers alive a bit longer, - # e.g., to drain in-batch requests. - self._create_standby_groups() - self._switch_and_prepare() + self._commit_scale_down(removing=False) + self._commit_new_dp_group() self._update_parallel_config() self.state = ScaleDownRemainingEngineState.COMPLETE + self._send_reconfigure_finished() return True else: @@ -402,26 +266,11 @@ class ElasticEPScalingState: def _progress_removing_engine(self) -> bool: state = self.state - assert self.old_dp_group is not None and self.old_dp_store is not None + assert self.old_dp_group is not None if state == ScaleDownRemovingEngineState.PREPARE: - self.state = ScaleDownRemovingEngineState.EPLB_RESHUFFLE - self.old_dp_store.add("eep_barrier_engine_count", 1) - return True - - if state == ScaleDownRemovingEngineState.EPLB_RESHUFFLE: - if ( - int(self.old_dp_store.get("eep_barrier_engine_count")) - < self.old_dp_group.size() - ): - return False - if not self._staged_barrier( - use_new_group=False, barrier_name="eplb_reshuffle" - ): - return False assert self.old_dp_group.rank() > 0 - self._eplb_reshuffle_before_scale_down() - self._switch_and_remove() + self._commit_scale_down(removing=True) self.state = ScaleDownRemovingEngineState.COMPLETE self.engine_core._eep_send_engine_core_notification( EEPNotificationType.SHUTDOWN_COMPLETE @@ -432,22 +281,22 @@ class ElasticEPScalingState: assert self.state == ScaleDownRemovingEngineState.COMPLETE return True - def handle_notification(self, notification_type: EEPNotificationType): - assert self.worker_type != "new" - assert self.old_dp_store is not None - if ( - notification_type == EEPNotificationType.NEW_CORE_ENGINES_INIT_READY - and self.state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT - ): - self.old_dp_store.add("eep_barrier_engine_count", 1) - self.state = ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS - elif ( - notification_type == EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY - and self.state - == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT - ): - self.old_dp_store.add("eep_barrier_engine_count", 1) - self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS + def is_ready_for_switch(self) -> bool: + return self.worker_type == "existing" and ( + self.state is ScaleUpExistingEngineState.COMMIT_SCALE_UP + or self.state is ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN + ) + + @property + def ready_key(self) -> str: + return f"eep_ready/{self.engine_core.dp_rank}" + + def _mark_ready_for_switch(self) -> None: + parallel_config = self.new_parallel_config + get_cached_tcp_store_client( + parallel_config.data_parallel_master_ip, + parallel_config._coord_store_port, + ).set(self.ready_key, b"1") def is_complete(self) -> bool: if self.scale_type == "scale_up": @@ -462,50 +311,78 @@ class ElasticEPScalingState: else self.state == ScaleDownRemainingEngineState.COMPLETE ) - def _create_standby_groups(self): + def _init_new_dp_group(self) -> tuple[Any, Any]: + return self.new_parallel_config.stateless_init_dp_group(return_store=True) + + def _ensure_new_dp_group(self) -> bool: + if self.new_dp_group is not None: + return True + + if self._prepare_future is None: + self._prepare_future = self._prepare_executor.submit( + self._init_new_dp_group + ) + if not self._prepare_future.done(): + return False + + self.new_dp_group, self.new_dp_store = self._prepare_future.result() + self._prepare_future = None + return True + + def _create_standby_groups(self) -> bool: assert self.old_dp_group is not None - self.new_dp_group, self.new_dp_store = ( - self.new_parallel_config.stateless_init_dp_group(return_store=True) - ) - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("create_standby_groups", self.reconfig_request) - ) + if not self._ensure_new_dp_group(): + return False + if not self._execute_async( + "create_standby_groups", + self.reconfig_request, + self.new_parallel_config.use_all2all, + ): + return False if self.old_dp_group.rank() == 0: logger.info("[Elastic EP] Created standby communication groups") + return True - def _transfer_weights(self): + def _transfer_weights(self) -> bool: assert self.reconfig_request is not None and self.old_dp_group is not None old_dp_size = self.old_dp_group.size() new_dp_size = self.reconfig_request.new_data_parallel_size - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("transfer_weights", old_dp_size, new_dp_size) - ) + if not self._execute_async("transfer_weights", old_dp_size, new_dp_size): + return False if self.old_dp_group.rank() == 0: logger.info("[Elastic EP] Transferred weights to new workers") + return True - def _transfer_expert_mapping(self): - assert self.old_dp_group is not None - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("broadcast_expert_mapping",) - ) - if self.old_dp_group.rank() == 0: - logger.info("[Elastic EP] Broadcasted expert mapping to new workers") - - def _sync_kv_cache_memory_size(self): + def _sync_kv_cache_memory_size(self) -> bool: assert self.engine_core.available_gpu_memory_for_kv_cache > 0 assert self.new_dp_group is not None and self.old_dp_group is not None - ParallelConfig.sync_kv_cache_memory_size( - self.new_dp_group, - self.engine_core.available_gpu_memory_for_kv_cache, - ) + + if self._new_dp_sync is None: + tensor = torch.tensor( + [self.engine_core.available_gpu_memory_for_kv_cache], + dtype=torch.int64, + device="cpu", + ) + work = torch.distributed.all_reduce( + tensor, + op=torch.distributed.ReduceOp.MIN, + group=self.new_dp_group, + async_op=True, + ) + self._new_dp_sync = (tensor, work) + return False + + _, work = self._new_dp_sync + if not work.is_completed(): + return False + work.wait() + self._new_dp_sync = None if self.old_dp_group.rank() == 0: logger.info("[Elastic EP] Synced KV cache memory size to new workers") + return True - def _switch_and_prepare(self): - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("switch_and_prepare",) - ) + def _commit_new_dp_group(self): old_dp_group = self.old_dp_group stateless_destroy_torch_distributed_process_group(old_dp_group) assert self.new_dp_group is not None @@ -529,41 +406,28 @@ class ElasticEPScalingState: self.engine_core.current_wave = int(data[1]) self.engine_core.step_counter = int(data[2]) if new_dp_group.rank() == 0: + logger.info("[Elastic EP] Switched to new setup") + + def _send_reconfigure_finished(self): + assert self.new_dp_group is not None + if self.new_dp_group.rank() == 0: self.engine_core._eep_send_engine_core_notification( EEPNotificationType.RECONFIGURE_FINISHED ) - logger.info("[Elastic EP] Switched to new setup") - def _eplb_reshuffle(self): - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("perform_eplb_reshuffle",) - ) - # Reshuffle changes per-rank token routing; the locked MoE workspace - # may now be too small. Rewarm covers both new and existing engines. - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("rewarm_workspace",) - ) - assert self.new_dp_group is not None - if self.new_dp_group.rank() == 0: - logger.info("[Elastic EP] EPLB reshuffle completed") - - def _eplb_reshuffle_before_scale_down(self): + def _commit_scale_down(self, removing: bool): assert self.reconfig_request is not None and self.old_dp_group is not None - self.model_executor.collective_rpc( + self._collective_rpc( "elastic_ep_execute", args=( - "perform_scale_down_eplb_reshuffle", + "commit_scale_down", self.reconfig_request.new_data_parallel_size, + removing, ), ) if self.old_dp_group.rank() == 0: logger.info("[Elastic EP] EPLB reshuffle completed") - def _switch_and_remove(self): - self.model_executor.collective_rpc( - "elastic_ep_execute", args=("switch_and_remove",) - ) - def _update_parallel_config(self): assert self.reconfig_request is not None reconfig_request = self.reconfig_request diff --git a/vllm/distributed/elastic_ep/standby_state.py b/vllm/distributed/elastic_ep/standby_state.py index 846793a955..1892f3e794 100644 --- a/vllm/distributed/elastic_ep/standby_state.py +++ b/vllm/distributed/elastic_ep/standby_state.py @@ -39,6 +39,7 @@ def create_standby_groups( new_world_size_across_dp: int, master_ip: str, coord_store_port: int, + use_all2all: bool, enable_eplb: bool = True, backend: str | None = None, ) -> None: @@ -86,7 +87,7 @@ def create_standby_groups( ) standby_ep_ranks = [x.tolist() for x in standby_ep_ranks] _STANDBY_EP = _init_stateless_group( - standby_ep_ranks, "ep", master_ip, backend, coord_store=coord_store + standby_ep_ranks, "ep", master_ip, backend, coord_store, use_all2all=use_all2all ) if enable_eplb: diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 4284a609d6..a90e8acbca 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -414,6 +414,7 @@ class GroupCoordinator: use_device_communicator: bool, # whether to use device communicator use_message_queue_broadcaster: bool = False, group_name: str | None = None, + use_all2all: bool = False, ): group_name = group_name or "anonymous" self.unique_name = _get_unique_name(group_name) @@ -508,6 +509,7 @@ class GroupCoordinator: device=self.device, device_group=self.device_group, unique_name=self.unique_name, + use_all2all=use_all2all, ) from vllm.distributed.device_communicators.shm_broadcast import MessageQueue @@ -1321,6 +1323,7 @@ def init_model_parallel_group( use_message_queue_broadcaster: bool = False, group_name: str | None = None, use_device_communicator: bool = True, + use_all2all: bool = False, ) -> GroupCoordinator: return GroupCoordinator( group_ranks=group_ranks, @@ -1329,6 +1332,7 @@ def init_model_parallel_group( use_device_communicator=use_device_communicator, use_message_queue_broadcaster=use_message_queue_broadcaster, group_name=group_name, + use_all2all=use_all2all, ) @@ -1339,6 +1343,7 @@ def _init_stateless_group( backend: str, coord_store: Store, use_device_communicator: bool = True, + use_all2all: bool = False, ) -> "StatelessGroupCoordinator": """Create a StatelessGroupCoordinator with the given parameters.""" from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator @@ -1354,6 +1359,7 @@ def _init_stateless_group( coord_store=coord_store, global_rank=world.rank, global_world_size=world.world_size, + use_all2all=use_all2all, ) @@ -1924,6 +1930,7 @@ def initialize_model_parallel( .unbind(0) ) group_ranks = [x.tolist() for x in group_ranks] + use_all2all = parallel_config.use_all2all if enable_elastic_ep: _EP = _init_stateless_group( group_ranks, @@ -1931,10 +1938,15 @@ def initialize_model_parallel( parallel_config.data_parallel_master_ip, backend, coord_store=coord_store, + use_all2all=use_all2all, ) else: _EP = init_model_parallel_group( - group_ranks, get_world_group().local_rank, backend, group_name="ep" + group_ranks, + get_world_group().local_rank, + backend, + group_name="ep", + use_all2all=use_all2all, ) # Create EPLB group with the same ranks as EP if EPLB is enabled. diff --git a/vllm/distributed/stateless_coordinator.py b/vllm/distributed/stateless_coordinator.py index 5f4597d07c..38c74a97c5 100644 --- a/vllm/distributed/stateless_coordinator.py +++ b/vllm/distributed/stateless_coordinator.py @@ -79,6 +79,7 @@ class StatelessGroupCoordinator(GroupCoordinator): host: str = "127.0.0.1", global_rank: int = 0, global_world_size: int = 1, + use_all2all: bool = False, ): group_name = group_name or "anonymous" self.unique_name = _get_unique_name(group_name) @@ -191,6 +192,7 @@ class StatelessGroupCoordinator(GroupCoordinator): global_ranks=self.ranks, global_world_size=global_world_size, tcp_store_group=self.tcp_store_group, + use_all2all=use_all2all, ) self.mq_broadcaster = None diff --git a/vllm/entrypoints/serve/elastic_ep/api_router.py b/vllm/entrypoints/serve/elastic_ep/api_router.py index e711a257dd..02a2425090 100644 --- a/vllm/entrypoints/serve/elastic_ep/api_router.py +++ b/vllm/entrypoints/serve/elastic_ep/api_router.py @@ -12,10 +12,7 @@ from vllm.engine.protocol import EngineClient from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, ) -from vllm.entrypoints.serve.elastic_ep.middleware import ( - get_scaling_elastic_ep, - set_scaling_elastic_ep, -) +from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.logger import init_logger @@ -64,8 +61,6 @@ async def scale_elastic_ep(raw_request: Request): status_code=400, detail="drain_timeout must be a positive integer" ) - # Set scaling flag to prevent new requests - set_scaling_elastic_ep(True) client = engine_client(raw_request) try: await client.scale_elastic_ep(new_data_parallel_size, drain_timeout) @@ -83,8 +78,6 @@ async def scale_elastic_ep(raw_request: Request): except Exception as e: logger.error("Scale failed: %s", e) raise HTTPException(status_code=500, detail="Scale failed") from e - finally: - set_scaling_elastic_ep(False) @router.post("/is_scaling_elastic_ep") diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index e461dae0bb..b2c989b295 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -93,7 +93,7 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None: ) -def kernel_warmup(worker: "Worker"): +def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): from vllm.model_executor.warmup.minimax_m3_msa_warmup import ( minimax_m3_msa_warmup, ) @@ -118,6 +118,22 @@ def kernel_warmup(worker: "Worker"): ) # Run next so input-prep kernels JIT against pristine runner state. + if worker.vllm_config.kernel_config.enable_jit_warmup: + fa4_cutedsl_warmup(worker) + sparse_mla_triton_warmup(worker) + + if current_platform.has_device_capability(90): + _warmup_ll_bf16_router_gemm(worker.get_model()) + + if worker.vllm_config.kernel_config.enable_cutedsl_warmup: + # TODO(roberto): Remove after registered CuTeDSL warmups are migrated + # to the shared JIT warmup infrastructure. + # https://github.com/vllm-project/vllm/pull/47451 + cutedsl_warmup() + + if process_local_only: + return + flashinfer_sparse_mla_decode_autotune_warmup(worker) deepseek_v4_sparse_mla_attention_warmup(worker) @@ -143,9 +159,6 @@ def kernel_warmup(worker: "Worker"): elif has_flashinfer() and current_platform.has_device_capability(90): flashinfer_autotune(worker.model_runner) - if current_platform.has_device_capability(90): - _warmup_ll_bf16_router_gemm(worker.get_model()) - # FlashInfer attention warmup # Only warmup if the model has FlashInfer attention groups # and is not a pooling model @@ -178,16 +191,6 @@ def kernel_warmup(worker: "Worker"): create_mixed_batch=True, ) - if worker.vllm_config.kernel_config.enable_cutedsl_warmup: - # TODO(roberto): Remove after registered CuTeDSL warmups are migrated - # to the shared JIT warmup infrastructure. - # https://github.com/vllm-project/vllm/pull/47451 - cutedsl_warmup() - - if worker.vllm_config.kernel_config.enable_jit_warmup: - fa4_cutedsl_warmup(worker) - sparse_mla_triton_warmup(worker) - def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None: if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None: diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index e80be0e45d..83033ecf81 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -35,8 +35,6 @@ FT_STATUS_CALL_ID = -2 class EEPNotificationType(enum.Enum): - NEW_CORE_ENGINES_INIT_READY = "NEW_CORE_ENGINES_INIT_READY" - NEW_CORE_ENGINES_WEIGHTS_INIT_READY = "NEW_CORE_ENGINES_WEIGHTS_INIT_READY" RECONFIGURE_FINISHED = "RECONFIGURE_FINISHED" SHUTDOWN_COMPLETE = "SHUTDOWN_COMPLETE" diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 5c2e01cf44..e8d33c961d 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -109,6 +109,7 @@ class AsyncLLM(EngineClient): maybe_register_config_serialize_by_value() self.vllm_config = vllm_config + self._elastic_ep_lock = asyncio.Lock() self.model_config = vllm_config.model_config self.observability_config = vllm_config.observability_config @@ -998,17 +999,27 @@ class AsyncLLM(EngineClient): "waiting for requests to drain." ) + async def _drain_requests_for_elastic_ep(self, drain_timeout: int) -> None: + try: + logger.info( + "VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, " + "waiting for requests to drain before scaling" + ) + await self.wait_for_requests_to_drain(drain_timeout) + except BaseException: + set_scaling_elastic_ep(False) + raise + async def scale_elastic_ep( self, new_data_parallel_size: int, drain_timeout: int = 300 ): - """ - Scale up or down the data parallel size by adding or removing - engine cores. - Args: - new_data_parallel_size: The new number of data parallel workers - drain_timeout: - Maximum time to wait for requests to drain (seconds) - """ + """Scale the elastic EP data parallel size.""" + async with self._elastic_ep_lock: + await self._scale_elastic_ep(new_data_parallel_size, drain_timeout) + + async def _scale_elastic_ep( + self, new_data_parallel_size: int, drain_timeout: int + ) -> None: old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size if old_data_parallel_size == new_data_parallel_size: logger.info( @@ -1017,12 +1028,7 @@ class AsyncLLM(EngineClient): ) return - if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS: - logger.info( - "VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, " - "waiting for requests to drain before scaling" - ) - await self.wait_for_requests_to_drain(drain_timeout) + await self.engine_core.prepare_elastic_ep(new_data_parallel_size) # recreate stat loggers if new_data_parallel_size > old_data_parallel_size and self.log_stats: @@ -1042,11 +1048,12 @@ class AsyncLLM(EngineClient): self.logger_manager.log_engine_initialized() set_scaling_elastic_ep(True) - try: - await self.engine_core.scale_elastic_ep(new_data_parallel_size) - self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size - finally: - set_scaling_elastic_ep(False) + if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS: + await self._drain_requests_for_elastic_ep(drain_timeout) + + await self.engine_core.commit_elastic_ep() + self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size + set_scaling_elastic_ep(False) async def handle_fault( self, fault_tolerance_request: FaultToleranceRequest diff --git a/vllm/v1/engine/coordinator.py b/vllm/v1/engine/coordinator.py index 2f3b03636d..d7f05cffc8 100644 --- a/vllm/v1/engine/coordinator.py +++ b/vllm/v1/engine/coordinator.py @@ -376,6 +376,9 @@ class DPCoordinatorProc: eng_index = outputs.engine_index scheduler_stats = outputs.scheduler_stats if scheduler_stats: + # Elastic EP stats may arrive while the engine list changes. + if eng_index >= len(self.engines): + continue # 1. Updated request load stats - update our local # state with these. stats = self.engines[eng_index].request_counts diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 9917f810b5..ecac92f5fe 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -327,8 +327,9 @@ class EngineCore: vllm_config.validate_block_size() - # Initialize kv cache and warmup the execution self.model_executor.initialize_from_config(kv_cache_configs) + if not envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: + self.model_executor.compile_or_warm_up_model() elapsed = time.time() - start compile_time = vllm_config.compilation_config.compilation_time @@ -993,9 +994,7 @@ class EngineCore: raise NotImplementedError def _eep_send_engine_core_notification( - self, - notification_type: EEPNotificationType, - vllm_config: VllmConfig | None = None, + self, notification_type: EEPNotificationType ): raise NotImplementedError @@ -1070,11 +1069,6 @@ class EngineCoreProc(EngineCore): self.addresses = addresses self.process_input_queue_block = True - if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: - self._eep_send_engine_core_notification( - EEPNotificationType.NEW_CORE_ENGINES_INIT_READY, - vllm_config=vllm_config, - ) self._init_data_parallel(vllm_config) super().__init__( @@ -2096,12 +2090,16 @@ class DPEngineCoreProc(EngineCoreProc): self._maybe_publish_request_counts() if self.eep_scaling_state is not None: - _ = self.eep_scaling_state.progress() - if self.eep_scaling_state.is_complete(): - if self.eep_scaling_state.worker_type == "removing": + state = self.eep_scaling_state + if state.commit_requested or not state.is_ready_for_switch(): + state.progress() + if state.is_complete(): + if state.worker_type == "removing": raise SystemExit self.process_input_queue_block = True self.eep_scaling_state = None + elif not state.commit_requested and state.is_ready_for_switch(): + self.process_input_queue_block = True executed = self._process_engine_step() self._maybe_publish_request_counts() @@ -2171,7 +2169,7 @@ class DPEngineCoreProc(EngineCoreProc): def reinitialize_distributed( self, reconfig_request: ReconfigureDistributedRequest - ) -> None: + ) -> str: from copy import deepcopy from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState @@ -2203,7 +2201,10 @@ class DPEngineCoreProc(EngineCoreProc): == ReconfigureRankType.SHUTDOWN_CURRENT_RANK ) - self.eep_scaling_state = ElasticEPScalingState( + if self.eep_scaling_state is not None: + raise RuntimeError("Elastic EP reconfiguration is already active") + + state = ElasticEPScalingState( model_executor=self.model_executor, engine_core=self, vllm_config=self.vllm_config, @@ -2212,30 +2213,34 @@ class DPEngineCoreProc(EngineCoreProc): scale_type="scale_down" if is_scale_down else "scale_up", reconfig_request=reconfig_request, ) + self.eep_scaling_state = state + self.process_input_queue_block = False logger.info( "[Elastic EP] Received reconfiguration request and starting scaling up/down" ) + return state.ready_key + + def commit_prepared_elastic_ep(self) -> None: + state = self.eep_scaling_state + if state is None or state.commit_requested or not state.is_ready_for_switch(): + raise RuntimeError("No prepared Elastic EP reconfiguration is ready") + state.commit_requested = True + self.process_input_queue_block = False + logger.info("[Elastic EP] Committing prepared reconfiguration") def _eep_send_engine_core_notification( - self, - notification_type: EEPNotificationType, - vllm_config: VllmConfig | None = None, + self, notification_type: EEPNotificationType ): """ Send notifications to EngineCoreClient, which can then forward the notifications to other engine core processes. It is used for: - 1) In scale up: new core engines to notify existing core engines - that they are ready; - 2) In scale down: removing core engines to notify EngineCoreClient + 1) In scale down: removing core engines to notify EngineCoreClient so EngineCoreClient can release their ray placement groups; - 3) Both scale up/down: to notify EngineCoreClient that existing + 2) Both scale up/down: to notify EngineCoreClient that existing core engines have already switched to the new parallel setup. """ - if vllm_config is None: - dp_rank = self.vllm_config.parallel_config.data_parallel_rank - else: - dp_rank = vllm_config.parallel_config.data_parallel_rank + dp_rank = self.vllm_config.parallel_config.data_parallel_rank notification_data = (notification_type.value, dp_rank) outputs = EngineCoreOutputs( utility_output=UtilityOutput( @@ -2257,22 +2262,11 @@ class DPEngineCoreProc(EngineCoreProc): ): socket.send_multipart(encoder.encode(outputs)) - def eep_handle_engine_core_notification( - self, notification_type: str | EEPNotificationType - ): - """ - Handle notification received from EngineCoreClient - (forwarded from new core engines). - """ - assert self.eep_scaling_state is not None - if isinstance(notification_type, str): - notification_type = EEPNotificationType(notification_type) - self.eep_scaling_state.handle_notification(notification_type) - def _eep_scale_up_before_kv_init(self): from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState - self.eep_scaling_state = ElasticEPScalingState( + self.ignore_start_dp_wave = True + state = ElasticEPScalingState( model_executor=self.model_executor, engine_core=self, vllm_config=self.vllm_config, @@ -2281,7 +2275,10 @@ class DPEngineCoreProc(EngineCoreProc): scale_type="scale_up", reconfig_request=None, ) - self.eep_scaling_state.run_pre_kv_init_states() + if self.eep_scaling_state is not None: + raise RuntimeError("Elastic EP reconfiguration is already active") + self.eep_scaling_state = state + state.run_pre_kv_init_states() self.process_input_queue_block = False diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index a6c232b2ab..9460fdf48f 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -225,7 +225,10 @@ class EngineCoreClient(ABC): running state.""" raise NotImplementedError - async def scale_elastic_ep(self, new_data_parallel_size: int) -> None: + async def commit_elastic_ep(self) -> None: + raise NotImplementedError + + async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None: raise NotImplementedError async def get_output_async(self) -> EngineCoreOutputs: @@ -1490,6 +1493,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient): ) assert len(self.core_engines) > 1 + self._prepared_elastic_ep: tuple[int, int] | None = None self.eng_start_index = ( len(self.core_engines) * self.client_index @@ -1603,31 +1607,16 @@ class DPLBAsyncMPClient(DPAsyncMPClient): if len(cache.pending_notifications[notification_type]) >= abs( cache.num_new_core_engines ): - if notification_type == EEPNotificationType.SHUTDOWN_COMPLETE: - assert isinstance(self.resources.engine_manager, CoreEngineActorManager) - assert cache.num_new_core_engines < 0 - old_dp_size = len(cache.existing_core_engines) - new_dp_size = old_dp_size + cache.num_new_core_engines - self.resources.engine_manager.scale_down_elastic_ep( - old_dp_size, new_dp_size - ) - else: - await asyncio.gather( - *[ - self._call_utility_async( - "eep_handle_engine_core_notification", - notification_type, - engine=engine, - ) - for engine in cache.existing_core_engines - ] - ) - cache.pending_notifications[notification_type] = set() - if notification_type in [ - EEPNotificationType.SHUTDOWN_COMPLETE, - EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY, - ]: - self.eep_scaling_cache = None + engine_manager = self.resources.engine_manager + assert isinstance(engine_manager, CoreEngineActorManager) + assert cache.num_new_core_engines < 0 + old_dp_size = len(cache.existing_core_engines) + new_dp_size = old_dp_size + cache.num_new_core_engines + engine_manager.scale_down_elastic_ep(old_dp_size, new_dp_size) + self.vllm_config.parallel_config.data_parallel_size_local = len( + engine_manager.local_engine_actors + ) + self.eep_scaling_cache = None async def abort_requests_async(self, request_ids: list[str]) -> None: if not request_ids or self.resources.engine_dead: @@ -1651,31 +1640,46 @@ class DPLBAsyncMPClient(DPAsyncMPClient): ) -> None: await self._send_input(EngineCoreRequestType.ABORT, request_ids, engine) - async def scale_elastic_ep(self, new_data_parallel_size: int) -> None: - """Scale elastic EP data parallel size""" + async def commit_elastic_ep(self) -> None: + """Commit prepared elastic EP scaling.""" + prepared = self._prepared_elastic_ep + if prepared is None: + raise RuntimeError("Elastic EP scaling has not been prepared") + new_data_parallel_size, num_redundant_experts = prepared cur_data_parallel_size = len(self.core_engines) - - assert new_data_parallel_size != cur_data_parallel_size, ( - f"new_data_parallel_size {new_data_parallel_size} must be " - f"different from cur_data_parallel_size {cur_data_parallel_size}" + if new_data_parallel_size > cur_data_parallel_size: + await self._commit_scale_up_elastic_ep(new_data_parallel_size) + else: + await self._commit_scale_down_elastic_ep(new_data_parallel_size) + self.vllm_config.parallel_config.eplb_config.num_redundant_experts = ( + num_redundant_experts ) + self._prepared_elastic_ep = None + async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None: + """Prepare elastic EP scaling without routing requests to new engines.""" + if (prepared := self._prepared_elastic_ep) is not None: + if prepared[0] == new_data_parallel_size: + return + raise RuntimeError("Elastic EP scaling is already prepared") + cur_data_parallel_size = len(self.core_engines) assert self.vllm_config.parallel_config.data_parallel_backend == "ray", ( "Only ray DP backend supports scaling elastic EP" ) - - scale_up = new_data_parallel_size > cur_data_parallel_size - - if scale_up: - await self._scale_up_elastic_ep( - cur_data_parallel_size, new_data_parallel_size - ) + parallel_config = self.vllm_config.parallel_config + num_experts = self.vllm_config.model_config.get_num_experts() + num_redundant_experts = ( + num_experts + parallel_config.eplb_config.num_redundant_experts + ) * new_data_parallel_size // cur_data_parallel_size - num_experts + if new_data_parallel_size < cur_data_parallel_size: + await self._prepare_scale_down_elastic_ep(new_data_parallel_size) else: - await self._scale_down_elastic_ep( - cur_data_parallel_size, new_data_parallel_size + await self._prepare_scale_up_elastic_ep( + new_data_parallel_size, num_redundant_experts ) + self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts - async def _eep_wait_for_setup_switch_complete(self) -> None: + def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future: """ Wait for core engines to switch to the new setup. @@ -1687,9 +1691,26 @@ class DPLBAsyncMPClient(DPAsyncMPClient): future = asyncio.get_running_loop().create_future() self.utility_results[EEP_NOTIFICATION_CALL_ID] = future self._ensure_output_queue_task() - await future + return future - def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]: + def _wait_for_new_engine_ready(self, new_core_engines: list[bytes]) -> None: + new_engine_identities = set(new_core_engines) + sync_input_socket = zmq.Socket.shadow(self.input_socket) + while new_engine_identities: + if not sync_input_socket.poll(timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000): + raise TimeoutError( + f"Timed out waiting for new engine core processes to " + f"start. Waited " + f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by " + f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the " + f"timeout, set the environment variable: " + f"VLLM_ENGINE_READY_TIMEOUT_S=" + ) + identity, payload = sync_input_socket.recv_multipart() + new_engine_identities.discard(identity) + self._apply_ready_response(payload) + + def _setup_elastic_ep_reconfig_bootstrap(self) -> None: from vllm.distributed.utils import create_tcp_store from vllm.utils.network_utils import get_open_ports_list @@ -1709,36 +1730,36 @@ class DPLBAsyncMPClient(DPAsyncMPClient): ) parallel_config._coord_store_port = store.port self._coord_store = store - return ip, store.port - async def _scale_up_elastic_ep( - self, cur_data_parallel_size: int, new_data_parallel_size: int - ) -> None: - """Scale up the data parallel size by creating new engine cores - and reconfiguring existing ones.""" - cur_data_parallel_size = len(self.core_engines) - - self.eep_scaling_cache = ElasticScalingCache( - existing_core_engines=self.core_engines.copy(), - num_new_core_engines=new_data_parallel_size - cur_data_parallel_size, - pending_notifications=dict(), + def _make_reconfig_request( + self, + new_data_parallel_size: int, + rank_type: ReconfigureRankType = ReconfigureRankType.KEEP_CURRENT_RANK, + ) -> ReconfigureDistributedRequest: + parallel_config = self.vllm_config.parallel_config + return ReconfigureDistributedRequest( + new_data_parallel_size=new_data_parallel_size, + new_data_parallel_rank=rank_type, + new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK, + new_data_parallel_master_ip=parallel_config.data_parallel_master_ip, + new_data_parallel_master_port=parallel_config.data_parallel_master_port, + new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list, + coord_store_port=parallel_config._coord_store_port, ) - parallel_config = self.vllm_config.parallel_config - ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap() + async def _prepare_scale_up_elastic_ep( + self, + new_data_parallel_size: int, + num_redundant_experts: int, + ) -> None: + """Prepare scale up by creating new engine cores and reconfiguring + existing ones.""" + self._setup_elastic_ep_reconfig_bootstrap() # Phase 1: Send reconfig messages to existing engines reconfig_futures = [] for engine in self.core_engines: - reconfig_request = ReconfigureDistributedRequest( - new_data_parallel_size=new_data_parallel_size, - new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK, - new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK, - new_data_parallel_master_ip=ip, - new_data_parallel_master_port=parallel_config.data_parallel_master_port, - new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list, - coord_store_port=coord_store_port, - ) + reconfig_request = self._make_reconfig_request(new_data_parallel_size) coro = self._call_utility_async( "reinitialize_distributed", reconfig_request, engine=engine ) @@ -1746,51 +1767,54 @@ class DPLBAsyncMPClient(DPAsyncMPClient): # Phase 2: Create new engines assert isinstance(self.resources.engine_manager, CoreEngineActorManager) - parallel_config.eplb_config.num_redundant_experts = 0 start_new_worker_future = asyncio.to_thread( self.resources.engine_manager.scale_up_elastic_ep, self.vllm_config, new_data_parallel_size, + num_redundant_experts, ) - wait_future = self._eep_wait_for_setup_switch_complete() # Phase 3: Wait for new engines to be created # and reconfig messages to be received await asyncio.gather(start_new_worker_future, *reconfig_futures) + ready_keys = [future.result() for future in reconfig_futures] + ready_keys.extend( + f"eep_ready/{rank}" + for rank in range(len(self.core_engines), new_data_parallel_size) + ) + await asyncio.to_thread(self._coord_store.wait, ready_keys) logger.info("[Elastic EP] Successfully started new engines") - # Create new CoreEngine objects for the new engines - new_engine_identities = set() - for i in range(cur_data_parallel_size, new_data_parallel_size): - new_engine = i.to_bytes(2, "little") - self.core_engines.append(new_engine) - # NOTE(yongji): we don't update lb_engines here, - # we let run_engine_stats_update_task to update it. - new_engine_identities.add(new_engine) + async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None: + new_core_engines = [ + rank.to_bytes(2, "little") + for rank in range(len(self.core_engines), new_data_parallel_size) + ] - # Wait for ready messages from new engines on the input socket - sync_input_socket = zmq.Socket.shadow(self.input_socket) - while new_engine_identities: - if not sync_input_socket.poll( - timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000 # convert to ms - ): - raise TimeoutError( - f"Timed out waiting for new engine core processes to " - f"start. Waited " - f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by " - f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the " - f"timeout, set the environment variable: " - f"VLLM_ENGINE_READY_TIMEOUT_S=" - ) - identity, payload = sync_input_socket.recv_multipart() - new_engine_identities.discard(identity) - self._apply_ready_response(payload) + await self.pause_scheduler_async(mode="keep", clear_cache=False) + wait_future = self._eep_wait_for_setup_switch_complete() + finish_futures = [ + asyncio.create_task( + self._call_utility_async("commit_prepared_elastic_ep", engine=engine) + ) + for engine in self.core_engines + ] + try: + await asyncio.gather(*finish_futures) + await wait_future + self._wait_for_new_engine_ready(new_core_engines) + except Exception: + wait_future.cancel() + raise - # NOTE(yongji): Before we schedule any requests on the new workers, - # we should wait for them to switch to the new setup. - await wait_future + self.core_engines.extend(new_core_engines) # Update the parallel config - self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size + parallel_config = self.vllm_config.parallel_config + parallel_config.data_parallel_size = new_data_parallel_size + if isinstance(self.resources.engine_manager, CoreEngineActorManager): + parallel_config.data_parallel_size_local = len( + self.resources.engine_manager.local_engine_actors + ) # Notify coordinator about scale up through existing # stats_update_task connection self._ensure_stats_update_task() @@ -1803,10 +1827,23 @@ class DPLBAsyncMPClient(DPAsyncMPClient): "[Elastic EP] Scale up completed, new data parallel size: %s", new_data_parallel_size, ) + await self.resume_scheduler_async() - async def _scale_down_elastic_ep( - self, cur_data_parallel_size: int, new_data_parallel_size: int - ) -> None: + async def _prepare_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None: + self._setup_elastic_ep_reconfig_bootstrap() + + reconfig_futures = [] + for engine in self.core_engines[:new_data_parallel_size]: + reconfig_request = self._make_reconfig_request(new_data_parallel_size) + coro = self._call_utility_async( + "reinitialize_distributed", reconfig_request, engine=engine + ) + reconfig_futures.append(asyncio.create_task(coro)) + + ready_keys = await asyncio.gather(*reconfig_futures) + await asyncio.to_thread(self._coord_store.wait, ready_keys) + + async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None: """Scale down the data parallel size by shutting down and reconfiguring existing engine cores.""" cur_data_parallel_size = len(self.core_engines) @@ -1817,50 +1854,51 @@ class DPLBAsyncMPClient(DPAsyncMPClient): pending_notifications=dict(), ) - parallel_config = self.vllm_config.parallel_config - ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap() - + old_core_engines = self.core_engines + # NOTE(yongji): Immediately stop sending requests to the removing engines. + self.core_engines = old_core_engines[:new_data_parallel_size] + self.lb_engines = self.lb_engines[:new_data_parallel_size] removed_dp_size = cur_data_parallel_size - new_data_parallel_size + pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size + pause_futures = [ + self._call_utility_async("pause_scheduler", mode, False, engine=engine) + for mode, engine in zip(pause_modes, old_core_engines) + ] + await asyncio.gather(*pause_futures) assert isinstance(self.resources.engine_manager, CoreEngineActorManager) self.resources.engine_manager.remove_run_refs_for_scale_down(removed_dp_size) + wait_future = self._eep_wait_for_setup_switch_complete() reconfig_futures = [] - for cur_dp_rank, engine in enumerate(self.core_engines): - reconfig_request = ReconfigureDistributedRequest( - new_data_parallel_size=new_data_parallel_size, - new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK, - new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK, - new_data_parallel_master_ip=ip, - new_data_parallel_master_port=parallel_config.data_parallel_master_port, - new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list, - coord_store_port=coord_store_port, - ) - if cur_dp_rank >= new_data_parallel_size: - reconfig_request.new_data_parallel_rank = ( - ReconfigureRankType.SHUTDOWN_CURRENT_RANK + for cur_dp_rank, engine in enumerate(old_core_engines): + if cur_dp_rank < new_data_parallel_size: + coro = self._call_utility_async( + "commit_prepared_elastic_ep", engine=engine + ) + else: + reconfig_request = self._make_reconfig_request( + new_data_parallel_size, + ReconfigureRankType.SHUTDOWN_CURRENT_RANK, + ) + coro = self._call_utility_async( + "reinitialize_distributed", reconfig_request, engine=engine ) - coro = self._call_utility_async( - "reinitialize_distributed", reconfig_request, engine=engine - ) reconfig_futures.append(asyncio.create_task(coro)) - # NOTE(yongji): Immediately stop sending requests to the removing engines. - self.core_engines = self.core_engines[:new_data_parallel_size] - self.lb_engines = self.lb_engines[:new_data_parallel_size] - wait_future = self._eep_wait_for_setup_switch_complete() + try: + await asyncio.gather(*reconfig_futures) - await asyncio.gather(*reconfig_futures) + self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size + self._ensure_stats_update_task() + scale_down_marker = msgspec.msgpack.encode( + ("SCALE_ELASTIC_EP", new_data_parallel_size) + ) + await self.first_req_send_socket.send(scale_down_marker) + await wait_future + await self.resume_scheduler_async() + except Exception: + wait_future.cancel() + raise - self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size - self._ensure_stats_update_task() - scale_down_marker = msgspec.msgpack.encode( - ("SCALE_ELASTIC_EP", new_data_parallel_size) - ) - await self.first_req_send_socket.send(scale_down_marker) - - # NOTE(yongji): Unlike scaling up, - # here we don't actually need to wait for the setup switch to complete. - # We may want to remove it in the future. - await wait_future logger.info( "[Elastic EP] Scale down completed, new data parallel size: %s", new_data_parallel_size, diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index db1896b094..9b3bea0db9 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -821,7 +821,10 @@ class CoreEngineActorManager: return placement_groups, local_dp_ranks def scale_up_elastic_ep( - self, cur_vllm_config: VllmConfig, new_data_parallel_size: int + self, + cur_vllm_config: VllmConfig, + new_data_parallel_size: int, + num_redundant_experts: int, ) -> None: import copy @@ -864,6 +867,9 @@ class CoreEngineActorManager: if new_data_parallel_size > 1: _apply_dp_identity_suffix(dp_vllm_config, rank) dp_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size + dp_vllm_config.parallel_config.eplb_config.num_redundant_experts = ( + num_redundant_experts + ) dp_vllm_config.parallel_config.placement_group = pg # Check if this placement group is on the head node @@ -906,39 +912,18 @@ class CoreEngineActorManager: self.created_placement_groups.append(pg) self.placement_group_is_local.append(local_client) - ray.get( - [ - actor.wait_for_init.remote() - for actor in ( - self.local_engine_actors[-new_local_engines:] - if new_local_engines > 0 - else [] - ) - + self.remote_engine_actors[ - -(len(placement_groups) - new_local_engines) : - ] - ] - ) - actors = ( self.local_engine_actors[-new_local_engines:] if new_local_engines > 0 else [] ) + self.remote_engine_actors[-(len(placement_groups) - new_local_engines) :] + ray.get([actor.wait_for_init.remote() for actor in actors]) for actor in actors: ref = actor.run.remote() self.run_refs.append(ref) self.actor_run_ref_dict[actor] = ref - cur_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size - # Update old_vllm_config with new data_parallel_size_local if any new - # local engines were added - if new_local_engines > 0: - cur_vllm_config.parallel_config.data_parallel_size_local += ( - new_local_engines - ) - def scale_down_elastic_ep( self, cur_data_parallel_size: int, new_data_parallel_size: int ) -> None: diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index 4063844d46..404acd50de 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -116,11 +116,11 @@ class Executor(ABC): raise NotImplementedError def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None: - """ - Initialize the KV caches and begin the model execution loop of the - underlying workers. - """ + """Initialize the KV caches on the underlying workers.""" self.collective_rpc("initialize_from_config", args=(kv_cache_configs,)) + + def compile_or_warm_up_model(self) -> None: + """Compile/warm up the model and capture cudagraphs on workers.""" compilation_times: list[CompilationTimes] = self.collective_rpc( "compile_or_warm_up_model" )