[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -52,4 +52,5 @@ steps:
|
||||
- vllm/compilation/
|
||||
- tests/distributed/
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- pytest -v -s distributed/test_elastic_ep.py
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
@@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _traffic_loop(
|
||||
server: RemoteOpenAIServer,
|
||||
dp_rank: int | None,
|
||||
ready: threading.Barrier,
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
request_payload = None if is_probe else payload
|
||||
responses = []
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code))
|
||||
if status_code == 200:
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
if finished.is_set():
|
||||
return responses
|
||||
time.sleep(0.05)
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
|
||||
rejected = [end for _, end, status in responses if status == 503]
|
||||
if not rejected:
|
||||
return 0
|
||||
recovered = next(
|
||||
end for _, end, status in responses if status == 200 and end > rejected[-1]
|
||||
)
|
||||
return recovered - rejected[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
server: RemoteOpenAIServer,
|
||||
source_dp_size: int,
|
||||
new_dp_size: int,
|
||||
traffic_mode: str,
|
||||
) -> None:
|
||||
traffic_clients: list[int | None] = []
|
||||
if traffic_mode == "light":
|
||||
traffic_clients = [0]
|
||||
elif traffic_mode == "heavy":
|
||||
traffic_clients = [None] * source_dp_size
|
||||
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
|
||||
ready = threading.Barrier(len(clients) + 1)
|
||||
stop = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
|
||||
futures = [
|
||||
executor.submit(
|
||||
_traffic_loop, server, rank, ready, stop, finished, is_probe
|
||||
)
|
||||
for rank, is_probe in clients
|
||||
]
|
||||
try:
|
||||
ready.wait(timeout=120)
|
||||
start_time = time.perf_counter()
|
||||
assert _send_scale_command(server, new_dp_size)
|
||||
scale_seconds = time.perf_counter() - start_time
|
||||
finished.set()
|
||||
probe_result, *results = [future.result(timeout=120) for future in futures]
|
||||
finally:
|
||||
stop.set()
|
||||
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_503 = [start for start, _, status in probe_result if status == 503]
|
||||
assert probe_503, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200 and start_time <= request_start and request_end < probe_503[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
|
||||
f"[traffic={traffic_mode}] "
|
||||
f"scale_seconds={scale_seconds:.3f} "
|
||||
f"downtime_seconds={_downtime(probe_result):.3f}"
|
||||
)
|
||||
|
||||
|
||||
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
assert server.port is not None
|
||||
result = evaluate_gsm8k(
|
||||
@@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
return accuracy
|
||||
|
||||
|
||||
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
@@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"true" if use_async_eplb else "false",
|
||||
"true",
|
||||
"--eplb-config.step_interval",
|
||||
"10",
|
||||
"300",
|
||||
"--eplb-config.window_size",
|
||||
"5",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
"2",
|
||||
str(dp_size),
|
||||
"--api-server-count",
|
||||
"1",
|
||||
"--disable-access-log-for-endpoints",
|
||||
"/is_scaling_elastic_ep",
|
||||
]
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
args.extend(["--data-parallel-address", leader_address])
|
||||
if enforce_eager:
|
||||
args.append("--enforce-eager")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
("enforce_eager", "traffic_mode"),
|
||||
[
|
||||
pytest.param(True, "none", id="enforce_eager_none"),
|
||||
pytest.param(True, "light", id="enforce_eager_light"),
|
||||
pytest.param(True, "heavy", id="enforce_eager_heavy"),
|
||||
pytest.param(False, "heavy", id="cuda_graphs_heavy"),
|
||||
],
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
|
||||
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
|
||||
assert target_dp_size > initial_dp_size
|
||||
vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
) as server:
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
|
||||
|
||||
assert _send_scale_command(server, 4)
|
||||
time.sleep(10)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)")
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial")
|
||||
|
||||
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
assert _send_scale_command(server, 2)
|
||||
time.sleep(5)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
|
||||
|
||||
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
@@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
def test_elastic_ep_scaling_uneven():
|
||||
"""Test scale up with uneven worker distribution.
|
||||
|
||||
This tests the case where num_new_workers % old_dp_size != 0,
|
||||
specifically 2 -> 3 where remainder = 1 % 2 = 1.
|
||||
This exercises the remainder handling in sender-receiver pairing.
|
||||
"""
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
vllm_serve_args = _base_serve_args()
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
@@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
# Scale 2 -> 3: This has remainder = 1 % 2 = 1
|
||||
# Tests uneven sender-receiver pairing
|
||||
assert _send_scale_command(server, 3)
|
||||
time.sleep(10)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
|
||||
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
@@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
|
||||
# Scale back down to 2
|
||||
assert _send_scale_command(server, 2)
|
||||
time.sleep(5)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
|
||||
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
|
||||
@@ -686,6 +686,14 @@ class ParallelConfig:
|
||||
and self.data_parallel_size > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def use_all2all(self) -> bool:
|
||||
return (
|
||||
self.data_parallel_size > 1
|
||||
or self.use_sequence_parallel_moe
|
||||
or (self.enable_expert_parallel and self.prefill_context_parallel_size > 1)
|
||||
)
|
||||
|
||||
@property
|
||||
def use_batched_dp_moe(self) -> bool:
|
||||
return (
|
||||
@@ -786,6 +794,7 @@ class ParallelConfig:
|
||||
"data_parallel_master_ip",
|
||||
"data_parallel_master_port",
|
||||
"_data_parallel_master_port_list",
|
||||
"_coord_store_port",
|
||||
"data_parallel_rpc_port",
|
||||
"rank",
|
||||
"master_addr",
|
||||
|
||||
@@ -175,6 +175,7 @@ class DeviceCommunicatorBase:
|
||||
unique_name: str = "",
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
self.device = device or torch.device("cpu")
|
||||
self.cpu_group = cpu_group
|
||||
@@ -204,26 +205,15 @@ class DeviceCommunicatorBase:
|
||||
self.global_world_size = dist.get_world_size()
|
||||
self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank)
|
||||
|
||||
use_ep = False
|
||||
all2all_backend = None
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
|
||||
config = get_current_vllm_config_or_none()
|
||||
if config is not None:
|
||||
# initialize the all2all manager for DP or sequence-parallel EP.
|
||||
parallel_config = config.parallel_config
|
||||
use_ep = (
|
||||
parallel_config.data_parallel_size > 1
|
||||
or parallel_config.use_sequence_parallel_moe
|
||||
or (
|
||||
parallel_config.enable_expert_parallel
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
)
|
||||
)
|
||||
all2all_backend = parallel_config.all2all_backend
|
||||
all2all_backend = config.parallel_config.all2all_backend
|
||||
|
||||
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
||||
self.use_all2all = self.is_ep_communicator and use_ep
|
||||
self.use_all2all = self.is_ep_communicator and use_all2all
|
||||
self.all2all_backend = all2all_backend
|
||||
self.all2all_manager: All2AllManagerBase | None = None
|
||||
|
||||
|
||||
@@ -24,8 +24,11 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.dist_module = torch.distributed
|
||||
|
||||
if (
|
||||
|
||||
@@ -36,6 +36,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
global_ranks: list[int] | None = None,
|
||||
global_world_size: int | None = None,
|
||||
tcp_store_group: StatelessProcessGroup | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
cpu_group,
|
||||
@@ -44,6 +45,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
unique_name,
|
||||
global_ranks,
|
||||
global_world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
if "tp" not in unique_name:
|
||||
# custom allreduce or torch symm mem can be used only by tp
|
||||
|
||||
@@ -20,8 +20,11 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(cpu_group, device, device_group, unique_name)
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.ca_comm: None = None
|
||||
if self.use_all2all:
|
||||
if self.all2all_backend in ("naive", "allgather_reducescatter"):
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
import gc
|
||||
import weakref
|
||||
from collections.abc import Iterable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
|
||||
make_eep_staged_quant_method,
|
||||
)
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
@@ -145,6 +146,10 @@ class ElasticEPScalingExecutor:
|
||||
self.worker_ref = weakref.ref(worker)
|
||||
self.reconfig_request = None
|
||||
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
|
||||
self._async_executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="ElasticEPAsync"
|
||||
)
|
||||
self._async_future: Future[None] | None = None
|
||||
|
||||
@property
|
||||
def worker(self):
|
||||
@@ -159,59 +164,68 @@ class ElasticEPScalingExecutor:
|
||||
raise ValueError(f"Unknown execute method: {execute_method}")
|
||||
return method(*args, **kwargs)
|
||||
|
||||
def _set_eplb_suppressed(self, suppressed: bool) -> None:
|
||||
self.worker.model_runner.eep_eplb_suppressed = suppressed
|
||||
ep_group = get_standby_ep_group() or get_ep_group()
|
||||
if ep_group.rank == 0:
|
||||
logger.info(
|
||||
"[Elastic EP] EPLB %s elastic scaling transition",
|
||||
"disabled during" if suppressed else "re-enabled after",
|
||||
)
|
||||
def start_async(self, execute_method: str, *args, **kwargs) -> str:
|
||||
if self._async_future is not None:
|
||||
raise RuntimeError("Another Elastic EP async method is active")
|
||||
if args and isinstance(args[0], ReconfigureDistributedRequest):
|
||||
self.reconfig_request = args[0]
|
||||
dp_rank = self.worker.vllm_config.parallel_config.data_parallel_rank
|
||||
done_key = f"eep_async/{execute_method}/{dp_rank}/{self.worker.rank}"
|
||||
self._async_future = self._async_executor.submit(
|
||||
self._run_async, execute_method, *args, **kwargs
|
||||
)
|
||||
self._async_future.add_done_callback(lambda _: self._mark_async_done(done_key))
|
||||
return done_key
|
||||
|
||||
def _run_async(self, execute_method: str, *args, **kwargs) -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
self.worker.vllm_config.enable_trace_function_call_for_thread()
|
||||
assert hasattr(self.worker, "device")
|
||||
current_platform.set_device(self.worker.device)
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
self.execute(execute_method, *args, **kwargs)
|
||||
|
||||
def _mark_async_done(self, done_key: str) -> None:
|
||||
from vllm.distributed.utils import get_cached_tcp_store_client
|
||||
|
||||
assert self.reconfig_request is not None
|
||||
get_cached_tcp_store_client(
|
||||
self.reconfig_request.new_data_parallel_master_ip,
|
||||
self.reconfig_request.coord_store_port,
|
||||
).set(done_key, b"1")
|
||||
|
||||
def clear_async(self) -> None:
|
||||
future = self._async_future
|
||||
if future is None:
|
||||
raise RuntimeError("No Elastic EP async method is active")
|
||||
if not future.done():
|
||||
raise RuntimeError("Elastic EP async method is not done")
|
||||
self._async_future = None
|
||||
future.result()
|
||||
|
||||
def load_model(self) -> None:
|
||||
(
|
||||
expanded_physical_to_logical,
|
||||
num_logical_experts,
|
||||
old_num_physical_experts,
|
||||
) = self.receive_expert_mapping()
|
||||
num_physical_experts = expanded_physical_to_logical.shape[1]
|
||||
self.worker.parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_physical_experts - num_logical_experts
|
||||
)
|
||||
self.worker.load_model(load_dummy_weights=True)
|
||||
self.worker.model_runner.setup_eplb_from_mapping(
|
||||
expanded_physical_to_logical, old_num_physical_experts
|
||||
)
|
||||
self._set_eplb_suppressed(True)
|
||||
|
||||
def create_standby_groups(
|
||||
self, reconfig_request: ReconfigureDistributedRequest
|
||||
self, reconfig_request: ReconfigureDistributedRequest, use_all2all: bool
|
||||
) -> None:
|
||||
self.reconfig_request = reconfig_request
|
||||
new_dp_size = reconfig_request.new_data_parallel_size
|
||||
old_dp_size = get_dp_group().world_size
|
||||
world_size = self.worker.vllm_config.parallel_config.world_size
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
world_size = parallel_config.world_size
|
||||
new_world_size_across_dp = world_size * new_dp_size
|
||||
updated_config = copy.copy(self.worker.vllm_config)
|
||||
updated_config.parallel_config = copy.deepcopy(
|
||||
self.worker.vllm_config.parallel_config
|
||||
create_standby_groups(
|
||||
new_dp_size=new_dp_size,
|
||||
new_world_size_across_dp=new_world_size_across_dp,
|
||||
master_ip=reconfig_request.new_data_parallel_master_ip,
|
||||
coord_store_port=reconfig_request.coord_store_port,
|
||||
use_all2all=use_all2all,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
)
|
||||
updated_config.parallel_config.data_parallel_size = new_dp_size
|
||||
with set_current_vllm_config(updated_config):
|
||||
create_standby_groups(
|
||||
new_dp_size=new_dp_size,
|
||||
new_world_size_across_dp=new_world_size_across_dp,
|
||||
master_ip=reconfig_request.new_data_parallel_master_ip,
|
||||
coord_store_port=reconfig_request.coord_store_port,
|
||||
enable_eplb=updated_config.parallel_config.enable_eplb,
|
||||
)
|
||||
if new_dp_size > old_dp_size:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
elif new_dp_size < old_dp_size:
|
||||
self._stage_standby_moe_quant_methods()
|
||||
if new_dp_size < old_dp_size:
|
||||
self.stage_standby_moe_quant_methods()
|
||||
|
||||
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
@@ -265,6 +279,7 @@ class ElasticEPScalingExecutor:
|
||||
model_config = self.worker.model_runner.model_config
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
assert eplb_state is not None
|
||||
eplb_state.drain_async()
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
num_physical_experts = physical_to_logical.shape[1]
|
||||
@@ -278,10 +293,6 @@ class ElasticEPScalingExecutor:
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
# New workers enter load_model after receiving the expert mapping.
|
||||
# Stage replacement MoE kernels before returning to the state machine
|
||||
# so existing ranks can participate in collective EP comm creation.
|
||||
self._stage_standby_moe_quant_methods()
|
||||
|
||||
def _make_eep_moe_config(self, module, dp_group, ep_group):
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
@@ -300,7 +311,7 @@ class ElasticEPScalingExecutor:
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
|
||||
def _stage_standby_moe_quant_methods(self) -> None:
|
||||
def stage_standby_moe_quant_methods(self) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
model = self.worker.model_runner.get_model()
|
||||
@@ -500,26 +511,6 @@ class ElasticEPScalingExecutor:
|
||||
compilation_counter.stock_torch_compile_count += 1
|
||||
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
|
||||
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
for bt in multi_block_table.block_tables:
|
||||
saved_block_tables.append(
|
||||
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
|
||||
)
|
||||
multi_block_table.clear()
|
||||
|
||||
unlock_workspace()
|
||||
self.worker.compile_or_warm_up_model()
|
||||
lock_workspace()
|
||||
|
||||
for bt, (saved_gpu, saved_cpu) in zip(
|
||||
multi_block_table.block_tables, saved_block_tables
|
||||
):
|
||||
bt.block_table.gpu.copy_(saved_gpu)
|
||||
bt.block_table.cpu.copy_(saved_cpu)
|
||||
if new_dp_size < old_dp_size:
|
||||
self._set_eplb_suppressed(False)
|
||||
|
||||
def _perform_eplb_reshuffle(
|
||||
self, rank_mapping: dict[int, int] | None = None
|
||||
) -> None:
|
||||
@@ -553,12 +544,25 @@ class ElasticEPScalingExecutor:
|
||||
if get_ep_group().rank == 0:
|
||||
logger.info("[Elastic EP] Expert resharding completed")
|
||||
|
||||
def perform_eplb_reshuffle(self) -> None:
|
||||
def commit_scale_up(self, is_existing_worker: bool) -> None:
|
||||
if is_existing_worker:
|
||||
self.broadcast_expert_mapping()
|
||||
self.switch_and_prepare()
|
||||
else:
|
||||
mapping, _, num_valid_experts = self.receive_expert_mapping()
|
||||
self.worker.model_runner.setup_eplb_from_mapping(mapping, num_valid_experts)
|
||||
self._perform_eplb_reshuffle()
|
||||
self._set_eplb_suppressed(False)
|
||||
self.warm_and_capture()
|
||||
|
||||
def commit_scale_down(self, new_dp_size: int, removing: bool) -> None:
|
||||
self.perform_scale_down_eplb_reshuffle(new_dp_size)
|
||||
if removing:
|
||||
self.switch_and_remove()
|
||||
else:
|
||||
self.switch_and_prepare()
|
||||
self.warm_and_capture()
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
@@ -599,12 +603,17 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
|
||||
model = self.worker.model_runner.get_model()
|
||||
expert_weights = [
|
||||
module.get_expert_weights()
|
||||
for module in model.modules()
|
||||
if is_moe_layer(module)
|
||||
]
|
||||
batch_transfer_weights(
|
||||
model=model,
|
||||
is_sender=False,
|
||||
peer_rank=sender_rank,
|
||||
dp_group=dp_group,
|
||||
expert_weights=model.expert_weights,
|
||||
expert_weights=expert_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -643,14 +652,17 @@ class ElasticEPScalingExecutor:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
|
||||
|
||||
def rewarm_workspace(self) -> None:
|
||||
def warmup_local_kernels(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
kernel_warmup(self.worker, process_local_only=True)
|
||||
|
||||
def warm_and_capture(self) -> None:
|
||||
# Must run on every DP sibling in lockstep: _dummy_run calls
|
||||
# coordinate_batch_across_dp whenever data_parallel_size > 1
|
||||
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
|
||||
|
||||
# Save and clear block tables so profile_run/compile_or_warm_up_model
|
||||
# don't write dummy slot mappings into real KV-cache blocks (mirrors
|
||||
# switch_and_prepare's pattern).
|
||||
# Save and clear block tables so the dummy MoE forward doesn't
|
||||
# write dummy slot mappings into real KV-cache blocks.
|
||||
multi_block_table = self.worker.model_runner.input_batch.block_table
|
||||
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
for bt in multi_block_table.block_tables:
|
||||
@@ -660,19 +672,16 @@ class ElasticEPScalingExecutor:
|
||||
multi_block_table.clear()
|
||||
|
||||
# _ensure_workspace_size allocates a fresh tensor on grow, leaving
|
||||
# captured CUDA graphs with stale data pointers; drop graphs before
|
||||
# re-warm so captures realign with the resized buffer.
|
||||
# any captured CUDA graph with a stale data pointer; drop graphs
|
||||
# before re-warm so captures realign with the resized buffer.
|
||||
self._release_cuda_graphs()
|
||||
unlock_workspace()
|
||||
|
||||
# Grow the MoE workspace at max_num_tokens.
|
||||
# compile_or_warm_up_model alone only exercises cudagraph-capture
|
||||
# sizes (≤64 tokens for this test) and leaves the workspace at
|
||||
# ~10-14 MB; the post-all-to-all per-rank token count under real
|
||||
# post-reshuffle routing needs hundreds of MB. Use _dummy_run
|
||||
# directly (rather than profile_run) with skip_eplb=True so dummy
|
||||
# routing doesn't pollute the just-rebalanced EPLB stats — same
|
||||
# convention compile_or_warm_up_model itself uses.
|
||||
# Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model
|
||||
# alone only exercises cudagraph-capture sizes and can leave the
|
||||
# workspace too small for post-reshuffle routing. Use _dummy_run
|
||||
# directly with skip_eplb=True so dummy routing doesn't pollute the
|
||||
# just-rebalanced EPLB stats.
|
||||
runner = self.worker.model_runner
|
||||
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
|
||||
self.worker.compile_or_warm_up_model()
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import enum
|
||||
import time
|
||||
import weakref
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any, Literal, TypeAlias
|
||||
|
||||
import torch.distributed
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed import (
|
||||
sched_yield,
|
||||
stateless_destroy_torch_distributed_process_group,
|
||||
)
|
||||
from vllm.distributed.utils import get_cached_tcp_store_client
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import (
|
||||
EEPNotificationType,
|
||||
@@ -31,35 +30,29 @@ WorkerType = Literal["existing", "new", "removing"]
|
||||
|
||||
|
||||
class ScaleUpExistingEngineState(enum.IntEnum):
|
||||
WAIT_NEW_CORE_ENGINES_INIT = 0
|
||||
CREATE_STANDBY_GROUPS = 1
|
||||
TRANSFER_EXPERT_MAPPING = 2
|
||||
WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT = 3
|
||||
TRANSFER_WEIGHTS = 4
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 5
|
||||
SWITCH_AND_PREPARE = 6
|
||||
EPLB_RESHUFFLE = 7
|
||||
COMPLETE = 8
|
||||
CREATE_STANDBY_GROUPS = 0
|
||||
STAGE_QUANT_METHODS = 1
|
||||
TRANSFER_WEIGHTS = 2
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 3
|
||||
COMMIT_SCALE_UP = 4 # Blocks forward passes.
|
||||
COMPLETE = 5
|
||||
|
||||
|
||||
class ScaleUpNewEngineState(enum.IntEnum):
|
||||
PRE_KV_INIT = 0
|
||||
PREPARE = 1
|
||||
EPLB_RESHUFFLE = 2
|
||||
COMPLETE = 3
|
||||
COMPLETE = 2
|
||||
|
||||
|
||||
class ScaleDownRemainingEngineState(enum.IntEnum):
|
||||
PREPARE = 0
|
||||
EPLB_RESHUFFLE = 1
|
||||
SWITCH_AND_PREPARE = 2
|
||||
COMPLETE = 3
|
||||
COMMIT_SCALE_DOWN = 1 # Blocks forward passes.
|
||||
COMPLETE = 2
|
||||
|
||||
|
||||
class ScaleDownRemovingEngineState(enum.IntEnum):
|
||||
PREPARE = 0
|
||||
EPLB_RESHUFFLE = 1
|
||||
COMPLETE = 2
|
||||
COMPLETE = 1
|
||||
|
||||
|
||||
EngineState: TypeAlias = (
|
||||
@@ -70,15 +63,6 @@ EngineState: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class _BarrierTimeoutError(RuntimeError):
|
||||
"""
|
||||
Exception raised for timeout
|
||||
in the first stage of our two-staged
|
||||
TCPStore based barrier to synchronize the
|
||||
execution of all engines in the DP group.
|
||||
"""
|
||||
|
||||
|
||||
class ElasticEPScalingState:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -94,20 +78,24 @@ class ElasticEPScalingState:
|
||||
self.engine_core_ref = weakref.ref(engine_core)
|
||||
self.vllm_config = vllm_config
|
||||
self.old_dp_group = self.engine_core.dp_group if worker_type != "new" else None
|
||||
self.old_dp_store = self.engine_core.dp_store if worker_type != "new" else None
|
||||
self.new_parallel_config: ParallelConfig = new_parallel_config
|
||||
self.new_dp_group = self.engine_core.dp_group if worker_type == "new" else None
|
||||
self.new_dp_store = self.engine_core.dp_store if worker_type == "new" else None
|
||||
self.worker_type = worker_type
|
||||
self.scale_type = scale_type
|
||||
self.reconfig_request = reconfig_request
|
||||
|
||||
self.commit_requested = False
|
||||
self._prepare_executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="ElasticEPPrepare"
|
||||
)
|
||||
self._prepare_future: Future[Any] | None = None
|
||||
self._new_dp_sync: tuple[object, Any] | None = None
|
||||
self.state: EngineState
|
||||
if scale_type == "scale_up":
|
||||
self.state = (
|
||||
ScaleUpNewEngineState.PRE_KV_INIT
|
||||
if worker_type == "new"
|
||||
else ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
|
||||
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
|
||||
)
|
||||
else:
|
||||
self.state = (
|
||||
@@ -130,6 +118,31 @@ class ElasticEPScalingState:
|
||||
raise RuntimeError("Engine core has been garbage collected")
|
||||
return engine_core
|
||||
|
||||
def _collective_rpc(self, *args, **kwargs):
|
||||
return self.model_executor.collective_rpc(*args, **kwargs)
|
||||
|
||||
def _execute_async(self, execute_method: str, *args) -> bool:
|
||||
if self._prepare_future is None:
|
||||
done_keys = self._collective_rpc(
|
||||
"elastic_ep_execute",
|
||||
args=("start_async", execute_method, *args),
|
||||
)
|
||||
assert self.reconfig_request is not None
|
||||
coord_store = get_cached_tcp_store_client(
|
||||
self.reconfig_request.new_data_parallel_master_ip,
|
||||
self.reconfig_request.coord_store_port,
|
||||
)
|
||||
self._prepare_future = self._prepare_executor.submit(
|
||||
coord_store.wait, done_keys
|
||||
)
|
||||
if not self._prepare_future.done():
|
||||
return False
|
||||
|
||||
self._prepare_future.result()
|
||||
self._collective_rpc("elastic_ep_execute", args=("clear_async",))
|
||||
self._prepare_future = None
|
||||
return True
|
||||
|
||||
def progress(self) -> bool:
|
||||
if self.scale_type == "scale_up":
|
||||
return (
|
||||
@@ -149,157 +162,43 @@ class ElasticEPScalingState:
|
||||
assert self.progress()
|
||||
assert self.state == ScaleUpNewEngineState.PREPARE
|
||||
|
||||
def _execute_tcp_store_barrier(
|
||||
self, dp_store, group_rank, group_size, barrier_id, timeout=None
|
||||
):
|
||||
arrival_key = f"arrival_{barrier_id}_{group_rank}"
|
||||
dp_store.set(arrival_key, b"1")
|
||||
|
||||
start_time = time.time()
|
||||
processes_arrived: set[int] = set()
|
||||
|
||||
while len(processes_arrived) < group_size:
|
||||
if (
|
||||
timeout is not None
|
||||
and time.time() - start_time > timeout.total_seconds()
|
||||
):
|
||||
raise _BarrierTimeoutError(
|
||||
f"Barrier timed out after {timeout.total_seconds()} seconds"
|
||||
)
|
||||
|
||||
for i in range(group_size):
|
||||
if i in processes_arrived:
|
||||
continue
|
||||
|
||||
key = f"arrival_{barrier_id}_{i}"
|
||||
present = dp_store.check([key])
|
||||
if present:
|
||||
processes_arrived.add(i)
|
||||
|
||||
if len(processes_arrived) < group_size:
|
||||
sched_yield()
|
||||
|
||||
def _staged_barrier(self, use_new_group: bool, barrier_name: str) -> bool:
|
||||
"""
|
||||
Execute a two-staged barrier to synchronize all engines in the DP group.
|
||||
|
||||
Some DP EngineCores may receive the reconfiguration notifications
|
||||
later than others, and already proceed to engine step (model forward)
|
||||
in the busy loop.
|
||||
In this case, EngineCores that already proceed to reconfiguration
|
||||
should skip reconfiguration and execute model forward for one more
|
||||
step, so in the next step, all EngineCores will be synchronized.
|
||||
We use a two-staged barrier to achieve this. The first time each
|
||||
EngineCore executes the barrier, if a timeout is reached before the
|
||||
barrier completes, that means some EngineCores have already entered
|
||||
engine step. The EngineCores that timed out will then proceed to
|
||||
engine step, and will synchronize with the other EngineCores in the
|
||||
next step with a barrier without timeout.
|
||||
"""
|
||||
dp_group = self.new_dp_group if use_new_group else self.old_dp_group
|
||||
dp_store = self.new_dp_store if use_new_group else self.old_dp_store
|
||||
assert dp_group is not None and dp_store is not None
|
||||
|
||||
group_rank = dp_group.rank()
|
||||
group_size = dp_group.size()
|
||||
barrier_id = f"eep_barrier_{barrier_name}"
|
||||
sync_key = f"{barrier_id}_sync"
|
||||
|
||||
# TODO(yongji): figure out appropriate timeout for the barrier
|
||||
timeout = None if dp_store.check([sync_key]) else timedelta(seconds=5)
|
||||
|
||||
try:
|
||||
self._execute_tcp_store_barrier(
|
||||
dp_store, group_rank, group_size, barrier_id, timeout=timeout
|
||||
)
|
||||
torch.distributed.barrier(dp_group)
|
||||
if group_rank == 0:
|
||||
dp_store.delete_key(sync_key)
|
||||
for i in range(group_size):
|
||||
dp_store.delete_key(f"arrival_{barrier_id}_{i}")
|
||||
return True
|
||||
except _BarrierTimeoutError as e:
|
||||
if timeout is None:
|
||||
raise RuntimeError("Unexpected timeout encountered") from e
|
||||
dp_store.compare_set(sync_key, "", b"1")
|
||||
return False
|
||||
|
||||
def _progress_existing_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT:
|
||||
return False
|
||||
|
||||
elif state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
|
||||
# NOTE(yongji): wait for all existing workers to receive the request
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
|
||||
if not self._create_standby_groups():
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="create_standby_groups"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._create_standby_groups()
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING
|
||||
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.TRANSFER_EXPERT_MAPPING:
|
||||
self._transfer_expert_mapping()
|
||||
self.state = ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
|
||||
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
|
||||
if not self._execute_async("stage_standby_moe_quant_methods"):
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT:
|
||||
return False
|
||||
|
||||
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
if not self._transfer_weights():
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="transfer_weights"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._transfer_weights()
|
||||
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE:
|
||||
self._sync_kv_cache_memory_size()
|
||||
self.state = ScaleUpExistingEngineState.SWITCH_AND_PREPARE
|
||||
if not self._sync_kv_cache_memory_size():
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.COMMIT_SCALE_UP
|
||||
self._mark_ready_for_switch()
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.SWITCH_AND_PREPARE:
|
||||
self._switch_and_prepare()
|
||||
self.state = ScaleUpExistingEngineState.EPLB_RESHUFFLE
|
||||
assert self.new_dp_store is not None
|
||||
self.new_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.EPLB_RESHUFFLE:
|
||||
assert self.new_dp_group is not None and self.new_dp_store is not None
|
||||
if (
|
||||
int(self.new_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.new_dp_group.size()
|
||||
):
|
||||
elif state == ScaleUpExistingEngineState.COMMIT_SCALE_UP:
|
||||
if not self.commit_requested:
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=True, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
if self.new_dp_group.rank() == 0:
|
||||
self.new_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._eplb_reshuffle()
|
||||
self._commit_new_dp_group()
|
||||
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", True))
|
||||
self.state = ScaleUpExistingEngineState.COMPLETE
|
||||
self._update_parallel_config()
|
||||
self._send_reconfigure_finished()
|
||||
return True
|
||||
|
||||
else:
|
||||
@@ -311,22 +210,17 @@ class ElasticEPScalingState:
|
||||
assert self.new_dp_group is not None and self.new_dp_store is not None
|
||||
|
||||
if state == ScaleUpNewEngineState.PRE_KV_INIT:
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("receive_weights",)
|
||||
)
|
||||
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
|
||||
self.engine_core.available_gpu_memory_for_kv_cache = (
|
||||
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("prepare_new_worker",)
|
||||
)
|
||||
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
|
||||
self.state = ScaleUpNewEngineState.PREPARE
|
||||
return True
|
||||
|
||||
elif state == ScaleUpNewEngineState.PREPARE:
|
||||
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
|
||||
self._mark_ready_for_switch()
|
||||
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
|
||||
torch.distributed.all_reduce(
|
||||
tensor,
|
||||
@@ -337,22 +231,7 @@ class ElasticEPScalingState:
|
||||
self.engine_core.engines_running = bool(data[0])
|
||||
self.engine_core.current_wave = int(data[1])
|
||||
self.engine_core.step_counter = int(data[2])
|
||||
self.state = ScaleUpNewEngineState.EPLB_RESHUFFLE
|
||||
self.new_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
elif state == ScaleUpNewEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.new_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.new_dp_group.size()
|
||||
):
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=True, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
assert self.new_dp_group.rank() > 0
|
||||
self._eplb_reshuffle()
|
||||
self._collective_rpc("elastic_ep_execute", args=("commit_scale_up", False))
|
||||
self.state = ScaleUpNewEngineState.COMPLETE
|
||||
return True
|
||||
|
||||
@@ -362,38 +241,23 @@ class ElasticEPScalingState:
|
||||
|
||||
def _progress_remaining_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleDownRemainingEngineState.PREPARE:
|
||||
self.state = ScaleDownRemainingEngineState.EPLB_RESHUFFLE
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
if self._create_standby_groups():
|
||||
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
|
||||
self._mark_ready_for_switch()
|
||||
return True
|
||||
return False
|
||||
|
||||
elif state == ScaleDownRemainingEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
elif state == ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN:
|
||||
if not self.commit_requested:
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
self.old_dp_store.delete_key("eep_barrier_engine_count")
|
||||
self._eplb_reshuffle_before_scale_down()
|
||||
self.state = ScaleDownRemainingEngineState.SWITCH_AND_PREPARE
|
||||
# NOTE(yongji): currently, after EPLB reshuffle
|
||||
# that redistributes experts to remaining workers, workers
|
||||
# to be removed will immediately initiate shutdown;
|
||||
# existing workers can no longer execute forward steps using
|
||||
# the old setup. In the future, we may keep
|
||||
# the removing workers alive a bit longer,
|
||||
# e.g., to drain in-batch requests.
|
||||
self._create_standby_groups()
|
||||
self._switch_and_prepare()
|
||||
self._commit_scale_down(removing=False)
|
||||
self._commit_new_dp_group()
|
||||
self._update_parallel_config()
|
||||
self.state = ScaleDownRemainingEngineState.COMPLETE
|
||||
self._send_reconfigure_finished()
|
||||
return True
|
||||
|
||||
else:
|
||||
@@ -402,26 +266,11 @@ class ElasticEPScalingState:
|
||||
|
||||
def _progress_removing_engine(self) -> bool:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None and self.old_dp_store is not None
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleDownRemovingEngineState.PREPARE:
|
||||
self.state = ScaleDownRemovingEngineState.EPLB_RESHUFFLE
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
return True
|
||||
|
||||
if state == ScaleDownRemovingEngineState.EPLB_RESHUFFLE:
|
||||
if (
|
||||
int(self.old_dp_store.get("eep_barrier_engine_count"))
|
||||
< self.old_dp_group.size()
|
||||
):
|
||||
return False
|
||||
if not self._staged_barrier(
|
||||
use_new_group=False, barrier_name="eplb_reshuffle"
|
||||
):
|
||||
return False
|
||||
assert self.old_dp_group.rank() > 0
|
||||
self._eplb_reshuffle_before_scale_down()
|
||||
self._switch_and_remove()
|
||||
self._commit_scale_down(removing=True)
|
||||
self.state = ScaleDownRemovingEngineState.COMPLETE
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.SHUTDOWN_COMPLETE
|
||||
@@ -432,22 +281,22 @@ class ElasticEPScalingState:
|
||||
assert self.state == ScaleDownRemovingEngineState.COMPLETE
|
||||
return True
|
||||
|
||||
def handle_notification(self, notification_type: EEPNotificationType):
|
||||
assert self.worker_type != "new"
|
||||
assert self.old_dp_store is not None
|
||||
if (
|
||||
notification_type == EEPNotificationType.NEW_CORE_ENGINES_INIT_READY
|
||||
and self.state == ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_INIT
|
||||
):
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
self.state = ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
|
||||
elif (
|
||||
notification_type == EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY
|
||||
and self.state
|
||||
== ScaleUpExistingEngineState.WAIT_NEW_CORE_ENGINES_WEIGHTS_INIT
|
||||
):
|
||||
self.old_dp_store.add("eep_barrier_engine_count", 1)
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
|
||||
def is_ready_for_switch(self) -> bool:
|
||||
return self.worker_type == "existing" and (
|
||||
self.state is ScaleUpExistingEngineState.COMMIT_SCALE_UP
|
||||
or self.state is ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
|
||||
)
|
||||
|
||||
@property
|
||||
def ready_key(self) -> str:
|
||||
return f"eep_ready/{self.engine_core.dp_rank}"
|
||||
|
||||
def _mark_ready_for_switch(self) -> None:
|
||||
parallel_config = self.new_parallel_config
|
||||
get_cached_tcp_store_client(
|
||||
parallel_config.data_parallel_master_ip,
|
||||
parallel_config._coord_store_port,
|
||||
).set(self.ready_key, b"1")
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
if self.scale_type == "scale_up":
|
||||
@@ -462,50 +311,78 @@ class ElasticEPScalingState:
|
||||
else self.state == ScaleDownRemainingEngineState.COMPLETE
|
||||
)
|
||||
|
||||
def _create_standby_groups(self):
|
||||
def _init_new_dp_group(self) -> tuple[Any, Any]:
|
||||
return self.new_parallel_config.stateless_init_dp_group(return_store=True)
|
||||
|
||||
def _ensure_new_dp_group(self) -> bool:
|
||||
if self.new_dp_group is not None:
|
||||
return True
|
||||
|
||||
if self._prepare_future is None:
|
||||
self._prepare_future = self._prepare_executor.submit(
|
||||
self._init_new_dp_group
|
||||
)
|
||||
if not self._prepare_future.done():
|
||||
return False
|
||||
|
||||
self.new_dp_group, self.new_dp_store = self._prepare_future.result()
|
||||
self._prepare_future = None
|
||||
return True
|
||||
|
||||
def _create_standby_groups(self) -> bool:
|
||||
assert self.old_dp_group is not None
|
||||
self.new_dp_group, self.new_dp_store = (
|
||||
self.new_parallel_config.stateless_init_dp_group(return_store=True)
|
||||
)
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("create_standby_groups", self.reconfig_request)
|
||||
)
|
||||
if not self._ensure_new_dp_group():
|
||||
return False
|
||||
if not self._execute_async(
|
||||
"create_standby_groups",
|
||||
self.reconfig_request,
|
||||
self.new_parallel_config.use_all2all,
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Created standby communication groups")
|
||||
return True
|
||||
|
||||
def _transfer_weights(self):
|
||||
def _transfer_weights(self) -> bool:
|
||||
assert self.reconfig_request is not None and self.old_dp_group is not None
|
||||
old_dp_size = self.old_dp_group.size()
|
||||
new_dp_size = self.reconfig_request.new_data_parallel_size
|
||||
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("transfer_weights", old_dp_size, new_dp_size)
|
||||
)
|
||||
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Transferred weights to new workers")
|
||||
return True
|
||||
|
||||
def _transfer_expert_mapping(self):
|
||||
assert self.old_dp_group is not None
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("broadcast_expert_mapping",)
|
||||
)
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Broadcasted expert mapping to new workers")
|
||||
|
||||
def _sync_kv_cache_memory_size(self):
|
||||
def _sync_kv_cache_memory_size(self) -> bool:
|
||||
assert self.engine_core.available_gpu_memory_for_kv_cache > 0
|
||||
assert self.new_dp_group is not None and self.old_dp_group is not None
|
||||
ParallelConfig.sync_kv_cache_memory_size(
|
||||
self.new_dp_group,
|
||||
self.engine_core.available_gpu_memory_for_kv_cache,
|
||||
)
|
||||
|
||||
if self._new_dp_sync is None:
|
||||
tensor = torch.tensor(
|
||||
[self.engine_core.available_gpu_memory_for_kv_cache],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
work = torch.distributed.all_reduce(
|
||||
tensor,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.new_dp_group,
|
||||
async_op=True,
|
||||
)
|
||||
self._new_dp_sync = (tensor, work)
|
||||
return False
|
||||
|
||||
_, work = self._new_dp_sync
|
||||
if not work.is_completed():
|
||||
return False
|
||||
work.wait()
|
||||
self._new_dp_sync = None
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Synced KV cache memory size to new workers")
|
||||
return True
|
||||
|
||||
def _switch_and_prepare(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("switch_and_prepare",)
|
||||
)
|
||||
def _commit_new_dp_group(self):
|
||||
old_dp_group = self.old_dp_group
|
||||
stateless_destroy_torch_distributed_process_group(old_dp_group)
|
||||
assert self.new_dp_group is not None
|
||||
@@ -529,41 +406,28 @@ class ElasticEPScalingState:
|
||||
self.engine_core.current_wave = int(data[1])
|
||||
self.engine_core.step_counter = int(data[2])
|
||||
if new_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Switched to new setup")
|
||||
|
||||
def _send_reconfigure_finished(self):
|
||||
assert self.new_dp_group is not None
|
||||
if self.new_dp_group.rank() == 0:
|
||||
self.engine_core._eep_send_engine_core_notification(
|
||||
EEPNotificationType.RECONFIGURE_FINISHED
|
||||
)
|
||||
logger.info("[Elastic EP] Switched to new setup")
|
||||
|
||||
def _eplb_reshuffle(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("perform_eplb_reshuffle",)
|
||||
)
|
||||
# Reshuffle changes per-rank token routing; the locked MoE workspace
|
||||
# may now be too small. Rewarm covers both new and existing engines.
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("rewarm_workspace",)
|
||||
)
|
||||
assert self.new_dp_group is not None
|
||||
if self.new_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] EPLB reshuffle completed")
|
||||
|
||||
def _eplb_reshuffle_before_scale_down(self):
|
||||
def _commit_scale_down(self, removing: bool):
|
||||
assert self.reconfig_request is not None and self.old_dp_group is not None
|
||||
self.model_executor.collective_rpc(
|
||||
self._collective_rpc(
|
||||
"elastic_ep_execute",
|
||||
args=(
|
||||
"perform_scale_down_eplb_reshuffle",
|
||||
"commit_scale_down",
|
||||
self.reconfig_request.new_data_parallel_size,
|
||||
removing,
|
||||
),
|
||||
)
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] EPLB reshuffle completed")
|
||||
|
||||
def _switch_and_remove(self):
|
||||
self.model_executor.collective_rpc(
|
||||
"elastic_ep_execute", args=("switch_and_remove",)
|
||||
)
|
||||
|
||||
def _update_parallel_config(self):
|
||||
assert self.reconfig_request is not None
|
||||
reconfig_request = self.reconfig_request
|
||||
|
||||
@@ -39,6 +39,7 @@ def create_standby_groups(
|
||||
new_world_size_across_dp: int,
|
||||
master_ip: str,
|
||||
coord_store_port: int,
|
||||
use_all2all: bool,
|
||||
enable_eplb: bool = True,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
@@ -86,7 +87,7 @@ def create_standby_groups(
|
||||
)
|
||||
standby_ep_ranks = [x.tolist() for x in standby_ep_ranks]
|
||||
_STANDBY_EP = _init_stateless_group(
|
||||
standby_ep_ranks, "ep", master_ip, backend, coord_store=coord_store
|
||||
standby_ep_ranks, "ep", master_ip, backend, coord_store, use_all2all=use_all2all
|
||||
)
|
||||
|
||||
if enable_eplb:
|
||||
|
||||
@@ -414,6 +414,7 @@ class GroupCoordinator:
|
||||
use_device_communicator: bool, # whether to use device communicator
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: str | None = None,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
group_name = group_name or "anonymous"
|
||||
self.unique_name = _get_unique_name(group_name)
|
||||
@@ -508,6 +509,7 @@ class GroupCoordinator:
|
||||
device=self.device,
|
||||
device_group=self.device_group,
|
||||
unique_name=self.unique_name,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
@@ -1321,6 +1323,7 @@ def init_model_parallel_group(
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: str | None = None,
|
||||
use_device_communicator: bool = True,
|
||||
use_all2all: bool = False,
|
||||
) -> GroupCoordinator:
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
@@ -1329,6 +1332,7 @@ def init_model_parallel_group(
|
||||
use_device_communicator=use_device_communicator,
|
||||
use_message_queue_broadcaster=use_message_queue_broadcaster,
|
||||
group_name=group_name,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
|
||||
@@ -1339,6 +1343,7 @@ def _init_stateless_group(
|
||||
backend: str,
|
||||
coord_store: Store,
|
||||
use_device_communicator: bool = True,
|
||||
use_all2all: bool = False,
|
||||
) -> "StatelessGroupCoordinator":
|
||||
"""Create a StatelessGroupCoordinator with the given parameters."""
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
@@ -1354,6 +1359,7 @@ def _init_stateless_group(
|
||||
coord_store=coord_store,
|
||||
global_rank=world.rank,
|
||||
global_world_size=world.world_size,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
|
||||
@@ -1924,6 +1930,7 @@ def initialize_model_parallel(
|
||||
.unbind(0)
|
||||
)
|
||||
group_ranks = [x.tolist() for x in group_ranks]
|
||||
use_all2all = parallel_config.use_all2all
|
||||
if enable_elastic_ep:
|
||||
_EP = _init_stateless_group(
|
||||
group_ranks,
|
||||
@@ -1931,10 +1938,15 @@ def initialize_model_parallel(
|
||||
parallel_config.data_parallel_master_ip,
|
||||
backend,
|
||||
coord_store=coord_store,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
else:
|
||||
_EP = init_model_parallel_group(
|
||||
group_ranks, get_world_group().local_rank, backend, group_name="ep"
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
backend,
|
||||
group_name="ep",
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
# Create EPLB group with the same ranks as EP if EPLB is enabled.
|
||||
|
||||
@@ -79,6 +79,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
host: str = "127.0.0.1",
|
||||
global_rank: int = 0,
|
||||
global_world_size: int = 1,
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
group_name = group_name or "anonymous"
|
||||
self.unique_name = _get_unique_name(group_name)
|
||||
@@ -191,6 +192,7 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
global_ranks=self.ranks,
|
||||
global_world_size=global_world_size,
|
||||
tcp_store_group=self.tcp_store_group,
|
||||
use_all2all=use_all2all,
|
||||
)
|
||||
|
||||
self.mq_broadcaster = None
|
||||
|
||||
@@ -12,10 +12,7 @@ from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
||||
get_scaling_elastic_ep,
|
||||
set_scaling_elastic_ep,
|
||||
)
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -64,8 +61,6 @@ async def scale_elastic_ep(raw_request: Request):
|
||||
status_code=400, detail="drain_timeout must be a positive integer"
|
||||
)
|
||||
|
||||
# Set scaling flag to prevent new requests
|
||||
set_scaling_elastic_ep(True)
|
||||
client = engine_client(raw_request)
|
||||
try:
|
||||
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
@@ -83,8 +78,6 @@ async def scale_elastic_ep(raw_request: Request):
|
||||
except Exception as e:
|
||||
logger.error("Scale failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Scale failed") from e
|
||||
finally:
|
||||
set_scaling_elastic_ep(False)
|
||||
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
|
||||
@@ -93,7 +93,7 @@ def _warmup_ll_bf16_router_gemm(model: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker"):
|
||||
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
from vllm.model_executor.warmup.minimax_m3_msa_warmup import (
|
||||
minimax_m3_msa_warmup,
|
||||
)
|
||||
@@ -118,6 +118,22 @@ def kernel_warmup(worker: "Worker"):
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
fa4_cutedsl_warmup(worker)
|
||||
sparse_mla_triton_warmup(worker)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
|
||||
# to the shared JIT warmup infrastructure.
|
||||
# https://github.com/vllm-project/vllm/pull/47451
|
||||
cutedsl_warmup()
|
||||
|
||||
if process_local_only:
|
||||
return
|
||||
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
@@ -143,9 +159,6 @@ def kernel_warmup(worker: "Worker"):
|
||||
elif has_flashinfer() and current_platform.has_device_capability(90):
|
||||
flashinfer_autotune(worker.model_runner)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
# FlashInfer attention warmup
|
||||
# Only warmup if the model has FlashInfer attention groups
|
||||
# and is not a pooling model
|
||||
@@ -178,16 +191,6 @@ def kernel_warmup(worker: "Worker"):
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
|
||||
# to the shared JIT warmup infrastructure.
|
||||
# https://github.com/vllm-project/vllm/pull/47451
|
||||
cutedsl_warmup()
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
fa4_cutedsl_warmup(worker)
|
||||
sparse_mla_triton_warmup(worker)
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
|
||||
|
||||
@@ -35,8 +35,6 @@ FT_STATUS_CALL_ID = -2
|
||||
|
||||
|
||||
class EEPNotificationType(enum.Enum):
|
||||
NEW_CORE_ENGINES_INIT_READY = "NEW_CORE_ENGINES_INIT_READY"
|
||||
NEW_CORE_ENGINES_WEIGHTS_INIT_READY = "NEW_CORE_ENGINES_WEIGHTS_INIT_READY"
|
||||
RECONFIGURE_FINISHED = "RECONFIGURE_FINISHED"
|
||||
SHUTDOWN_COMPLETE = "SHUTDOWN_COMPLETE"
|
||||
|
||||
|
||||
+26
-19
@@ -109,6 +109,7 @@ class AsyncLLM(EngineClient):
|
||||
maybe_register_config_serialize_by_value()
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self._elastic_ep_lock = asyncio.Lock()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
|
||||
@@ -998,17 +999,27 @@ class AsyncLLM(EngineClient):
|
||||
"waiting for requests to drain."
|
||||
)
|
||||
|
||||
async def _drain_requests_for_elastic_ep(self, drain_timeout: int) -> None:
|
||||
try:
|
||||
logger.info(
|
||||
"VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, "
|
||||
"waiting for requests to drain before scaling"
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
except BaseException:
|
||||
set_scaling_elastic_ep(False)
|
||||
raise
|
||||
|
||||
async def scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int = 300
|
||||
):
|
||||
"""
|
||||
Scale up or down the data parallel size by adding or removing
|
||||
engine cores.
|
||||
Args:
|
||||
new_data_parallel_size: The new number of data parallel workers
|
||||
drain_timeout:
|
||||
Maximum time to wait for requests to drain (seconds)
|
||||
"""
|
||||
"""Scale the elastic EP data parallel size."""
|
||||
async with self._elastic_ep_lock:
|
||||
await self._scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
|
||||
async def _scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int
|
||||
) -> None:
|
||||
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info(
|
||||
@@ -1017,12 +1028,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
return
|
||||
|
||||
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
|
||||
logger.info(
|
||||
"VLLM_ELASTIC_EP_DRAIN_REQUESTS is set, "
|
||||
"waiting for requests to drain before scaling"
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
await self.engine_core.prepare_elastic_ep(new_data_parallel_size)
|
||||
|
||||
# recreate stat loggers
|
||||
if new_data_parallel_size > old_data_parallel_size and self.log_stats:
|
||||
@@ -1042,11 +1048,12 @@ class AsyncLLM(EngineClient):
|
||||
self.logger_manager.log_engine_initialized()
|
||||
|
||||
set_scaling_elastic_ep(True)
|
||||
try:
|
||||
await self.engine_core.scale_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
finally:
|
||||
set_scaling_elastic_ep(False)
|
||||
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
|
||||
await self._drain_requests_for_elastic_ep(drain_timeout)
|
||||
|
||||
await self.engine_core.commit_elastic_ep()
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
set_scaling_elastic_ep(False)
|
||||
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
|
||||
@@ -376,6 +376,9 @@ class DPCoordinatorProc:
|
||||
eng_index = outputs.engine_index
|
||||
scheduler_stats = outputs.scheduler_stats
|
||||
if scheduler_stats:
|
||||
# Elastic EP stats may arrive while the engine list changes.
|
||||
if eng_index >= len(self.engines):
|
||||
continue
|
||||
# 1. Updated request load stats - update our local
|
||||
# state with these.
|
||||
stats = self.engines[eng_index].request_counts
|
||||
|
||||
+36
-39
@@ -327,8 +327,9 @@ class EngineCore:
|
||||
|
||||
vllm_config.validate_block_size()
|
||||
|
||||
# Initialize kv cache and warmup the execution
|
||||
self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
if not envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self.model_executor.compile_or_warm_up_model()
|
||||
|
||||
elapsed = time.time() - start
|
||||
compile_time = vllm_config.compilation_config.compilation_time
|
||||
@@ -993,9 +994,7 @@ class EngineCore:
|
||||
raise NotImplementedError
|
||||
|
||||
def _eep_send_engine_core_notification(
|
||||
self,
|
||||
notification_type: EEPNotificationType,
|
||||
vllm_config: VllmConfig | None = None,
|
||||
self, notification_type: EEPNotificationType
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1070,11 +1069,6 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
self.addresses = addresses
|
||||
self.process_input_queue_block = True
|
||||
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self._eep_send_engine_core_notification(
|
||||
EEPNotificationType.NEW_CORE_ENGINES_INIT_READY,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self._init_data_parallel(vllm_config)
|
||||
|
||||
super().__init__(
|
||||
@@ -2096,12 +2090,16 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
self._maybe_publish_request_counts()
|
||||
|
||||
if self.eep_scaling_state is not None:
|
||||
_ = self.eep_scaling_state.progress()
|
||||
if self.eep_scaling_state.is_complete():
|
||||
if self.eep_scaling_state.worker_type == "removing":
|
||||
state = self.eep_scaling_state
|
||||
if state.commit_requested or not state.is_ready_for_switch():
|
||||
state.progress()
|
||||
if state.is_complete():
|
||||
if state.worker_type == "removing":
|
||||
raise SystemExit
|
||||
self.process_input_queue_block = True
|
||||
self.eep_scaling_state = None
|
||||
elif not state.commit_requested and state.is_ready_for_switch():
|
||||
self.process_input_queue_block = True
|
||||
|
||||
executed = self._process_engine_step()
|
||||
self._maybe_publish_request_counts()
|
||||
@@ -2171,7 +2169,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
def reinitialize_distributed(
|
||||
self, reconfig_request: ReconfigureDistributedRequest
|
||||
) -> None:
|
||||
) -> str:
|
||||
from copy import deepcopy
|
||||
|
||||
from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState
|
||||
@@ -2203,7 +2201,10 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
== ReconfigureRankType.SHUTDOWN_CURRENT_RANK
|
||||
)
|
||||
|
||||
self.eep_scaling_state = ElasticEPScalingState(
|
||||
if self.eep_scaling_state is not None:
|
||||
raise RuntimeError("Elastic EP reconfiguration is already active")
|
||||
|
||||
state = ElasticEPScalingState(
|
||||
model_executor=self.model_executor,
|
||||
engine_core=self,
|
||||
vllm_config=self.vllm_config,
|
||||
@@ -2212,30 +2213,34 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
scale_type="scale_down" if is_scale_down else "scale_up",
|
||||
reconfig_request=reconfig_request,
|
||||
)
|
||||
self.eep_scaling_state = state
|
||||
|
||||
self.process_input_queue_block = False
|
||||
logger.info(
|
||||
"[Elastic EP] Received reconfiguration request and starting scaling up/down"
|
||||
)
|
||||
return state.ready_key
|
||||
|
||||
def commit_prepared_elastic_ep(self) -> None:
|
||||
state = self.eep_scaling_state
|
||||
if state is None or state.commit_requested or not state.is_ready_for_switch():
|
||||
raise RuntimeError("No prepared Elastic EP reconfiguration is ready")
|
||||
state.commit_requested = True
|
||||
self.process_input_queue_block = False
|
||||
logger.info("[Elastic EP] Committing prepared reconfiguration")
|
||||
|
||||
def _eep_send_engine_core_notification(
|
||||
self,
|
||||
notification_type: EEPNotificationType,
|
||||
vllm_config: VllmConfig | None = None,
|
||||
self, notification_type: EEPNotificationType
|
||||
):
|
||||
"""
|
||||
Send notifications to EngineCoreClient, which can then forward
|
||||
the notifications to other engine core processes. It is used for:
|
||||
1) In scale up: new core engines to notify existing core engines
|
||||
that they are ready;
|
||||
2) In scale down: removing core engines to notify EngineCoreClient
|
||||
1) In scale down: removing core engines to notify EngineCoreClient
|
||||
so EngineCoreClient can release their ray placement groups;
|
||||
3) Both scale up/down: to notify EngineCoreClient that existing
|
||||
2) Both scale up/down: to notify EngineCoreClient that existing
|
||||
core engines have already switched to the new parallel setup.
|
||||
"""
|
||||
if vllm_config is None:
|
||||
dp_rank = self.vllm_config.parallel_config.data_parallel_rank
|
||||
else:
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
dp_rank = self.vllm_config.parallel_config.data_parallel_rank
|
||||
notification_data = (notification_type.value, dp_rank)
|
||||
outputs = EngineCoreOutputs(
|
||||
utility_output=UtilityOutput(
|
||||
@@ -2257,22 +2262,11 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
):
|
||||
socket.send_multipart(encoder.encode(outputs))
|
||||
|
||||
def eep_handle_engine_core_notification(
|
||||
self, notification_type: str | EEPNotificationType
|
||||
):
|
||||
"""
|
||||
Handle notification received from EngineCoreClient
|
||||
(forwarded from new core engines).
|
||||
"""
|
||||
assert self.eep_scaling_state is not None
|
||||
if isinstance(notification_type, str):
|
||||
notification_type = EEPNotificationType(notification_type)
|
||||
self.eep_scaling_state.handle_notification(notification_type)
|
||||
|
||||
def _eep_scale_up_before_kv_init(self):
|
||||
from vllm.distributed.elastic_ep.elastic_state import ElasticEPScalingState
|
||||
|
||||
self.eep_scaling_state = ElasticEPScalingState(
|
||||
self.ignore_start_dp_wave = True
|
||||
state = ElasticEPScalingState(
|
||||
model_executor=self.model_executor,
|
||||
engine_core=self,
|
||||
vllm_config=self.vllm_config,
|
||||
@@ -2281,7 +2275,10 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
scale_type="scale_up",
|
||||
reconfig_request=None,
|
||||
)
|
||||
self.eep_scaling_state.run_pre_kv_init_states()
|
||||
if self.eep_scaling_state is not None:
|
||||
raise RuntimeError("Elastic EP reconfiguration is already active")
|
||||
self.eep_scaling_state = state
|
||||
state.run_pre_kv_init_states()
|
||||
self.process_input_queue_block = False
|
||||
|
||||
|
||||
|
||||
+174
-136
@@ -225,7 +225,10 @@ class EngineCoreClient(ABC):
|
||||
running state."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
@@ -1490,6 +1493,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
)
|
||||
|
||||
assert len(self.core_engines) > 1
|
||||
self._prepared_elastic_ep: tuple[int, int] | None = None
|
||||
|
||||
self.eng_start_index = (
|
||||
len(self.core_engines) * self.client_index
|
||||
@@ -1603,31 +1607,16 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
if len(cache.pending_notifications[notification_type]) >= abs(
|
||||
cache.num_new_core_engines
|
||||
):
|
||||
if notification_type == EEPNotificationType.SHUTDOWN_COMPLETE:
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
assert cache.num_new_core_engines < 0
|
||||
old_dp_size = len(cache.existing_core_engines)
|
||||
new_dp_size = old_dp_size + cache.num_new_core_engines
|
||||
self.resources.engine_manager.scale_down_elastic_ep(
|
||||
old_dp_size, new_dp_size
|
||||
)
|
||||
else:
|
||||
await asyncio.gather(
|
||||
*[
|
||||
self._call_utility_async(
|
||||
"eep_handle_engine_core_notification",
|
||||
notification_type,
|
||||
engine=engine,
|
||||
)
|
||||
for engine in cache.existing_core_engines
|
||||
]
|
||||
)
|
||||
cache.pending_notifications[notification_type] = set()
|
||||
if notification_type in [
|
||||
EEPNotificationType.SHUTDOWN_COMPLETE,
|
||||
EEPNotificationType.NEW_CORE_ENGINES_WEIGHTS_INIT_READY,
|
||||
]:
|
||||
self.eep_scaling_cache = None
|
||||
engine_manager = self.resources.engine_manager
|
||||
assert isinstance(engine_manager, CoreEngineActorManager)
|
||||
assert cache.num_new_core_engines < 0
|
||||
old_dp_size = len(cache.existing_core_engines)
|
||||
new_dp_size = old_dp_size + cache.num_new_core_engines
|
||||
engine_manager.scale_down_elastic_ep(old_dp_size, new_dp_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size_local = len(
|
||||
engine_manager.local_engine_actors
|
||||
)
|
||||
self.eep_scaling_cache = None
|
||||
|
||||
async def abort_requests_async(self, request_ids: list[str]) -> None:
|
||||
if not request_ids or self.resources.engine_dead:
|
||||
@@ -1651,31 +1640,46 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
) -> None:
|
||||
await self._send_input(EngineCoreRequestType.ABORT, request_ids, engine)
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale elastic EP data parallel size"""
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
"""Commit prepared elastic EP scaling."""
|
||||
prepared = self._prepared_elastic_ep
|
||||
if prepared is None:
|
||||
raise RuntimeError("Elastic EP scaling has not been prepared")
|
||||
new_data_parallel_size, num_redundant_experts = prepared
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
assert new_data_parallel_size != cur_data_parallel_size, (
|
||||
f"new_data_parallel_size {new_data_parallel_size} must be "
|
||||
f"different from cur_data_parallel_size {cur_data_parallel_size}"
|
||||
if new_data_parallel_size > cur_data_parallel_size:
|
||||
await self._commit_scale_up_elastic_ep(new_data_parallel_size)
|
||||
else:
|
||||
await self._commit_scale_down_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
self._prepared_elastic_ep = None
|
||||
|
||||
async def prepare_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Prepare elastic EP scaling without routing requests to new engines."""
|
||||
if (prepared := self._prepared_elastic_ep) is not None:
|
||||
if prepared[0] == new_data_parallel_size:
|
||||
return
|
||||
raise RuntimeError("Elastic EP scaling is already prepared")
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
assert self.vllm_config.parallel_config.data_parallel_backend == "ray", (
|
||||
"Only ray DP backend supports scaling elastic EP"
|
||||
)
|
||||
|
||||
scale_up = new_data_parallel_size > cur_data_parallel_size
|
||||
|
||||
if scale_up:
|
||||
await self._scale_up_elastic_ep(
|
||||
cur_data_parallel_size, new_data_parallel_size
|
||||
)
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
num_experts = self.vllm_config.model_config.get_num_experts()
|
||||
num_redundant_experts = (
|
||||
num_experts + parallel_config.eplb_config.num_redundant_experts
|
||||
) * new_data_parallel_size // cur_data_parallel_size - num_experts
|
||||
if new_data_parallel_size < cur_data_parallel_size:
|
||||
await self._prepare_scale_down_elastic_ep(new_data_parallel_size)
|
||||
else:
|
||||
await self._scale_down_elastic_ep(
|
||||
cur_data_parallel_size, new_data_parallel_size
|
||||
await self._prepare_scale_up_elastic_ep(
|
||||
new_data_parallel_size, num_redundant_experts
|
||||
)
|
||||
self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts
|
||||
|
||||
async def _eep_wait_for_setup_switch_complete(self) -> None:
|
||||
def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future:
|
||||
"""
|
||||
Wait for core engines to switch to the new setup.
|
||||
|
||||
@@ -1687,9 +1691,26 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.utility_results[EEP_NOTIFICATION_CALL_ID] = future
|
||||
self._ensure_output_queue_task()
|
||||
await future
|
||||
return future
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> tuple[str, int]:
|
||||
def _wait_for_new_engine_ready(self, new_core_engines: list[bytes]) -> None:
|
||||
new_engine_identities = set(new_core_engines)
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while new_engine_identities:
|
||||
if not sync_input_socket.poll(timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000):
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for new engine core processes to "
|
||||
f"start. Waited "
|
||||
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
|
||||
f"timeout, set the environment variable: "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
self._apply_ready_response(payload)
|
||||
|
||||
def _setup_elastic_ep_reconfig_bootstrap(self) -> None:
|
||||
from vllm.distributed.utils import create_tcp_store
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
|
||||
@@ -1709,36 +1730,36 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
)
|
||||
parallel_config._coord_store_port = store.port
|
||||
self._coord_store = store
|
||||
return ip, store.port
|
||||
|
||||
async def _scale_up_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
"""Scale up the data parallel size by creating new engine cores
|
||||
and reconfiguring existing ones."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
self.eep_scaling_cache = ElasticScalingCache(
|
||||
existing_core_engines=self.core_engines.copy(),
|
||||
num_new_core_engines=new_data_parallel_size - cur_data_parallel_size,
|
||||
pending_notifications=dict(),
|
||||
def _make_reconfig_request(
|
||||
self,
|
||||
new_data_parallel_size: int,
|
||||
rank_type: ReconfigureRankType = ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
) -> ReconfigureDistributedRequest:
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
return ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=rank_type,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=parallel_config.data_parallel_master_ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=parallel_config._coord_store_port,
|
||||
)
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
|
||||
async def _prepare_scale_up_elastic_ep(
|
||||
self,
|
||||
new_data_parallel_size: int,
|
||||
num_redundant_experts: int,
|
||||
) -> None:
|
||||
"""Prepare scale up by creating new engine cores and reconfiguring
|
||||
existing ones."""
|
||||
self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
# Phase 1: Send reconfig messages to existing engines
|
||||
reconfig_futures = []
|
||||
for engine in self.core_engines:
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=coord_store_port,
|
||||
)
|
||||
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
@@ -1746,51 +1767,54 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
# Phase 2: Create new engines
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
parallel_config.eplb_config.num_redundant_experts = 0
|
||||
start_new_worker_future = asyncio.to_thread(
|
||||
self.resources.engine_manager.scale_up_elastic_ep,
|
||||
self.vllm_config,
|
||||
new_data_parallel_size,
|
||||
num_redundant_experts,
|
||||
)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
|
||||
# Phase 3: Wait for new engines to be created
|
||||
# and reconfig messages to be received
|
||||
await asyncio.gather(start_new_worker_future, *reconfig_futures)
|
||||
ready_keys = [future.result() for future in reconfig_futures]
|
||||
ready_keys.extend(
|
||||
f"eep_ready/{rank}"
|
||||
for rank in range(len(self.core_engines), new_data_parallel_size)
|
||||
)
|
||||
await asyncio.to_thread(self._coord_store.wait, ready_keys)
|
||||
logger.info("[Elastic EP] Successfully started new engines")
|
||||
|
||||
# Create new CoreEngine objects for the new engines
|
||||
new_engine_identities = set()
|
||||
for i in range(cur_data_parallel_size, new_data_parallel_size):
|
||||
new_engine = i.to_bytes(2, "little")
|
||||
self.core_engines.append(new_engine)
|
||||
# NOTE(yongji): we don't update lb_engines here,
|
||||
# we let run_engine_stats_update_task to update it.
|
||||
new_engine_identities.add(new_engine)
|
||||
async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
new_core_engines = [
|
||||
rank.to_bytes(2, "little")
|
||||
for rank in range(len(self.core_engines), new_data_parallel_size)
|
||||
]
|
||||
|
||||
# Wait for ready messages from new engines on the input socket
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while new_engine_identities:
|
||||
if not sync_input_socket.poll(
|
||||
timeout=VLLM_ENGINE_READY_TIMEOUT_S * 1000 # convert to ms
|
||||
):
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for new engine core processes to "
|
||||
f"start. Waited "
|
||||
f"{VLLM_ENGINE_READY_TIMEOUT_S}s (configured by "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S). To increase the "
|
||||
f"timeout, set the environment variable: "
|
||||
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
self._apply_ready_response(payload)
|
||||
await self.pause_scheduler_async(mode="keep", clear_cache=False)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
finish_futures = [
|
||||
asyncio.create_task(
|
||||
self._call_utility_async("commit_prepared_elastic_ep", engine=engine)
|
||||
)
|
||||
for engine in self.core_engines
|
||||
]
|
||||
try:
|
||||
await asyncio.gather(*finish_futures)
|
||||
await wait_future
|
||||
self._wait_for_new_engine_ready(new_core_engines)
|
||||
except Exception:
|
||||
wait_future.cancel()
|
||||
raise
|
||||
|
||||
# NOTE(yongji): Before we schedule any requests on the new workers,
|
||||
# we should wait for them to switch to the new setup.
|
||||
await wait_future
|
||||
self.core_engines.extend(new_core_engines)
|
||||
# Update the parallel config
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
parallel_config.data_parallel_size = new_data_parallel_size
|
||||
if isinstance(self.resources.engine_manager, CoreEngineActorManager):
|
||||
parallel_config.data_parallel_size_local = len(
|
||||
self.resources.engine_manager.local_engine_actors
|
||||
)
|
||||
# Notify coordinator about scale up through existing
|
||||
# stats_update_task connection
|
||||
self._ensure_stats_update_task()
|
||||
@@ -1803,10 +1827,23 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
new_data_parallel_size,
|
||||
)
|
||||
await self.resume_scheduler_async()
|
||||
|
||||
async def _scale_down_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
async def _prepare_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
reconfig_futures = []
|
||||
for engine in self.core_engines[:new_data_parallel_size]:
|
||||
reconfig_request = self._make_reconfig_request(new_data_parallel_size)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
reconfig_futures.append(asyncio.create_task(coro))
|
||||
|
||||
ready_keys = await asyncio.gather(*reconfig_futures)
|
||||
await asyncio.to_thread(self._coord_store.wait, ready_keys)
|
||||
|
||||
async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale down the data parallel size by shutting down and
|
||||
reconfiguring existing engine cores."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
@@ -1817,50 +1854,51 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
pending_notifications=dict(),
|
||||
)
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
old_core_engines = self.core_engines
|
||||
# NOTE(yongji): Immediately stop sending requests to the removing engines.
|
||||
self.core_engines = old_core_engines[:new_data_parallel_size]
|
||||
self.lb_engines = self.lb_engines[:new_data_parallel_size]
|
||||
removed_dp_size = cur_data_parallel_size - new_data_parallel_size
|
||||
pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size
|
||||
pause_futures = [
|
||||
self._call_utility_async("pause_scheduler", mode, False, engine=engine)
|
||||
for mode, engine in zip(pause_modes, old_core_engines)
|
||||
]
|
||||
await asyncio.gather(*pause_futures)
|
||||
assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
|
||||
self.resources.engine_manager.remove_run_refs_for_scale_down(removed_dp_size)
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
reconfig_futures = []
|
||||
for cur_dp_rank, engine in enumerate(self.core_engines):
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=ip,
|
||||
new_data_parallel_master_port=parallel_config.data_parallel_master_port,
|
||||
new_data_parallel_master_port_list=parallel_config._data_parallel_master_port_list,
|
||||
coord_store_port=coord_store_port,
|
||||
)
|
||||
if cur_dp_rank >= new_data_parallel_size:
|
||||
reconfig_request.new_data_parallel_rank = (
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK
|
||||
for cur_dp_rank, engine in enumerate(old_core_engines):
|
||||
if cur_dp_rank < new_data_parallel_size:
|
||||
coro = self._call_utility_async(
|
||||
"commit_prepared_elastic_ep", engine=engine
|
||||
)
|
||||
else:
|
||||
reconfig_request = self._make_reconfig_request(
|
||||
new_data_parallel_size,
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK,
|
||||
)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
coro = self._call_utility_async(
|
||||
"reinitialize_distributed", reconfig_request, engine=engine
|
||||
)
|
||||
reconfig_futures.append(asyncio.create_task(coro))
|
||||
|
||||
# NOTE(yongji): Immediately stop sending requests to the removing engines.
|
||||
self.core_engines = self.core_engines[:new_data_parallel_size]
|
||||
self.lb_engines = self.lb_engines[:new_data_parallel_size]
|
||||
wait_future = self._eep_wait_for_setup_switch_complete()
|
||||
try:
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
self._ensure_stats_update_task()
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
await wait_future
|
||||
await self.resume_scheduler_async()
|
||||
except Exception:
|
||||
wait_future.cancel()
|
||||
raise
|
||||
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
self._ensure_stats_update_task()
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size)
|
||||
)
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
|
||||
# NOTE(yongji): Unlike scaling up,
|
||||
# here we don't actually need to wait for the setup switch to complete.
|
||||
# We may want to remove it in the future.
|
||||
await wait_future
|
||||
logger.info(
|
||||
"[Elastic EP] Scale down completed, new data parallel size: %s",
|
||||
new_data_parallel_size,
|
||||
|
||||
+8
-23
@@ -821,7 +821,10 @@ class CoreEngineActorManager:
|
||||
return placement_groups, local_dp_ranks
|
||||
|
||||
def scale_up_elastic_ep(
|
||||
self, cur_vllm_config: VllmConfig, new_data_parallel_size: int
|
||||
self,
|
||||
cur_vllm_config: VllmConfig,
|
||||
new_data_parallel_size: int,
|
||||
num_redundant_experts: int,
|
||||
) -> None:
|
||||
import copy
|
||||
|
||||
@@ -864,6 +867,9 @@ class CoreEngineActorManager:
|
||||
if new_data_parallel_size > 1:
|
||||
_apply_dp_identity_suffix(dp_vllm_config, rank)
|
||||
dp_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
dp_vllm_config.parallel_config.eplb_config.num_redundant_experts = (
|
||||
num_redundant_experts
|
||||
)
|
||||
dp_vllm_config.parallel_config.placement_group = pg
|
||||
|
||||
# Check if this placement group is on the head node
|
||||
@@ -906,39 +912,18 @@ class CoreEngineActorManager:
|
||||
self.created_placement_groups.append(pg)
|
||||
self.placement_group_is_local.append(local_client)
|
||||
|
||||
ray.get(
|
||||
[
|
||||
actor.wait_for_init.remote()
|
||||
for actor in (
|
||||
self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0
|
||||
else []
|
||||
)
|
||||
+ self.remote_engine_actors[
|
||||
-(len(placement_groups) - new_local_engines) :
|
||||
]
|
||||
]
|
||||
)
|
||||
|
||||
actors = (
|
||||
self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0
|
||||
else []
|
||||
) + self.remote_engine_actors[-(len(placement_groups) - new_local_engines) :]
|
||||
|
||||
ray.get([actor.wait_for_init.remote() for actor in actors])
|
||||
for actor in actors:
|
||||
ref = actor.run.remote()
|
||||
self.run_refs.append(ref)
|
||||
self.actor_run_ref_dict[actor] = ref
|
||||
|
||||
cur_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
# Update old_vllm_config with new data_parallel_size_local if any new
|
||||
# local engines were added
|
||||
if new_local_engines > 0:
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local += (
|
||||
new_local_engines
|
||||
)
|
||||
|
||||
def scale_down_elastic_ep(
|
||||
self, cur_data_parallel_size: int, new_data_parallel_size: int
|
||||
) -> None:
|
||||
|
||||
@@ -116,11 +116,11 @@ class Executor(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None:
|
||||
"""
|
||||
Initialize the KV caches and begin the model execution loop of the
|
||||
underlying workers.
|
||||
"""
|
||||
"""Initialize the KV caches on the underlying workers."""
|
||||
self.collective_rpc("initialize_from_config", args=(kv_cache_configs,))
|
||||
|
||||
def compile_or_warm_up_model(self) -> None:
|
||||
"""Compile/warm up the model and capture cudagraphs on workers."""
|
||||
compilation_times: list[CompilationTimes] = self.collective_rpc(
|
||||
"compile_or_warm_up_model"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user