Elastic Expert Parallel Initial Support (#20775)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
@@ -177,3 +177,19 @@ class EngineCoreRequestType(enum.Enum):
|
||||
UTILITY = b'\x03'
|
||||
# Sentinel used within EngineCoreProc.
|
||||
EXECUTOR_FAILED = b'\x04'
|
||||
|
||||
|
||||
class ReconfigureDistributedRequest(msgspec.Struct):
|
||||
new_data_parallel_size: int
|
||||
new_data_parallel_rank: int
|
||||
new_data_parallel_rank_local: int
|
||||
new_data_parallel_master_ip: str
|
||||
new_data_parallel_master_port: int
|
||||
|
||||
|
||||
class ReconfigureRankType(enum.IntEnum):
|
||||
"""
|
||||
Rank type for reconfiguring distributed request.
|
||||
"""
|
||||
KEEP_CURRENT_RANK = -1
|
||||
SHUTDOWN_CURRENT_RANK = -2
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from copy import copy
|
||||
from typing import Any, Optional, Union
|
||||
@@ -608,6 +609,63 @@ class AsyncLLM(EngineClient):
|
||||
return await self.engine_core.collective_rpc_async(
|
||||
method, timeout, args, kwargs)
|
||||
|
||||
async def wait_for_requests_to_drain(self, drain_timeout: int = 300):
|
||||
"""Wait for all requests to be drained."""
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < drain_timeout:
|
||||
if not self.engine_core.dp_engines_running():
|
||||
logger.info("Engines are idle, requests have been drained")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Engines are still running, waiting for requests to drain...")
|
||||
await asyncio.sleep(1) # Wait 1 second before checking again
|
||||
|
||||
raise TimeoutError(f"Timeout reached after {drain_timeout} seconds "
|
||||
"waiting for requests to drain.")
|
||||
|
||||
async def scale_elastic_ep(self,
|
||||
new_data_parallel_size: int,
|
||||
drain_timeout: int = 300):
|
||||
"""
|
||||
Scale up or down the data parallel size by adding or removing
|
||||
engine cores.
|
||||
Args:
|
||||
new_data_parallel_size: The new number of data parallel workers
|
||||
drain_timeout:
|
||||
Maximum time to wait for requests to drain (seconds)
|
||||
"""
|
||||
old_data_parallel_size = \
|
||||
self.vllm_config.parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info("Data parallel size is already %s, skipping scale",
|
||||
new_data_parallel_size)
|
||||
return
|
||||
logger.info(
|
||||
"Waiting for requests to drain before "
|
||||
"scaling up to %s engines...", new_data_parallel_size)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
logger.info(
|
||||
"Requests have been drained, proceeding with scale "
|
||||
"to %s engines", new_data_parallel_size)
|
||||
await self.engine_core.scale_elastic_ep(new_data_parallel_size)
|
||||
self.vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
|
||||
# recreate stat loggers
|
||||
if new_data_parallel_size > old_data_parallel_size:
|
||||
stat_loggers: list[list[StatLoggerBase]] = setup_default_loggers(
|
||||
vllm_config=self.vllm_config,
|
||||
log_stats=self.log_stats,
|
||||
engine_num=new_data_parallel_size,
|
||||
custom_stat_loggers=None,
|
||||
)
|
||||
num_new_engines = len(stat_loggers) - len(self.stat_loggers)
|
||||
self.stat_loggers.extend(stat_loggers[-num_new_engines:])
|
||||
else:
|
||||
for _ in range(old_data_parallel_size - new_data_parallel_size):
|
||||
self.stat_loggers.pop()
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
# Is None before the loop is started.
|
||||
|
||||
@@ -200,11 +200,41 @@ class CoordinatorProc:
|
||||
# Ignore subscription messages.
|
||||
continue
|
||||
|
||||
decoded = msgspec.msgpack.decode(buffer)
|
||||
if isinstance(decoded, (list, tuple)) and len(
|
||||
decoded) == 2 and decoded[0] == "SCALE_ELASTIC_EP":
|
||||
# Handle scale up notification
|
||||
new_engine_count = decoded[1]
|
||||
current_count = len(self.engines)
|
||||
if new_engine_count > current_count:
|
||||
for _ in range(new_engine_count - current_count):
|
||||
self.engines.append(EngineState())
|
||||
# NOTE(yongji): handle the case
|
||||
# where newly started engines have current_wave = 0
|
||||
# if existing engines just finished a wave
|
||||
# and engine_running isn't updated yet at
|
||||
# CoordinatorProc requests routed to newly started
|
||||
# engines may not wake up existing engines, as long
|
||||
# as 0 < request.wave < existing engines'
|
||||
# current_wave
|
||||
# we note that 0 is the wave number for the new
|
||||
# engine
|
||||
self.engines_running = False
|
||||
logger.info(
|
||||
"DPCoordinator scaled up from %s to %s "
|
||||
"engines", current_count, new_engine_count)
|
||||
else:
|
||||
self.engines = self.engines[:new_engine_count]
|
||||
logger.info(
|
||||
"DPCoordinator scaled down from %s to %s "
|
||||
"engines", current_count, new_engine_count)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = msgspec.msgpack.decode(buffer)
|
||||
engine_to_exclude, wave = decoded
|
||||
if not self.engines_running:
|
||||
if wave < self.current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
|
||||
+64
-5
@@ -32,7 +32,9 @@ from vllm.v1.core.sched.interface import SchedulerInterface
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler as V1Scheduler
|
||||
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
EngineCoreRequestType, UtilityOutput)
|
||||
EngineCoreRequestType,
|
||||
ReconfigureDistributedRequest, ReconfigureRankType,
|
||||
UtilityOutput)
|
||||
from vllm.v1.engine.mm_input_cache import MirroredProcessingCache
|
||||
from vllm.v1.engine.utils import EngineHandshakeMetadata, EngineZmqAddresses
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
@@ -77,6 +79,8 @@ class EngineCore:
|
||||
self.model_executor.register_failure_callback(
|
||||
executor_fail_callback)
|
||||
|
||||
self.available_gpu_memory_for_kv_cache = -1
|
||||
|
||||
# Setup KV Caches and update CacheConfig after profiling.
|
||||
num_gpu_blocks, num_cpu_blocks, kv_cache_config = \
|
||||
self._initialize_kv_caches(vllm_config)
|
||||
@@ -137,12 +141,23 @@ class EngineCore:
|
||||
# Get all kv cache needed by the model
|
||||
kv_cache_specs = self.model_executor.get_kv_cache_specs()
|
||||
|
||||
# Profiles the peak memory usage of the model to determine how much
|
||||
# memory can be allocated for kv cache.
|
||||
has_kv_cache = any(kv_cache_spec for kv_cache_spec in kv_cache_specs)
|
||||
if has_kv_cache:
|
||||
available_gpu_memory = \
|
||||
self.model_executor.determine_available_memory()
|
||||
if os.environ.get("VLLM_ELASTIC_EP_SCALE_UP_LAUNCH") == "1":
|
||||
dp_group = getattr(self, "dp_group", None)
|
||||
assert dp_group is not None
|
||||
self.available_gpu_memory_for_kv_cache = \
|
||||
ParallelConfig.sync_kv_cache_memory_size(dp_group, -1)
|
||||
available_gpu_memory = [
|
||||
self.available_gpu_memory_for_kv_cache
|
||||
] * len(kv_cache_specs)
|
||||
else:
|
||||
# Profiles the peak memory usage of the model to determine how
|
||||
# much memory can be allocated for kv cache.
|
||||
available_gpu_memory = (
|
||||
self.model_executor.determine_available_memory())
|
||||
self.available_gpu_memory_for_kv_cache = \
|
||||
available_gpu_memory[0]
|
||||
else:
|
||||
# Attention free models don't need memory for kv cache
|
||||
available_gpu_memory = [0] * len(kv_cache_specs)
|
||||
@@ -989,6 +1004,50 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
return ParallelConfig.has_unfinished_dp(self.dp_group,
|
||||
local_unfinished)
|
||||
|
||||
def reinitialize_distributed(
|
||||
self, reconfig_request: ReconfigureDistributedRequest) -> None:
|
||||
stateless_destroy_torch_distributed_process_group(self.dp_group)
|
||||
self.shutdown()
|
||||
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
old_dp_size = parallel_config.data_parallel_size
|
||||
parallel_config.data_parallel_size = \
|
||||
reconfig_request.new_data_parallel_size
|
||||
if reconfig_request.new_data_parallel_rank != -1:
|
||||
parallel_config.data_parallel_rank = \
|
||||
reconfig_request.new_data_parallel_rank
|
||||
# local rank specifies device visibility, it should not be changed
|
||||
assert reconfig_request.new_data_parallel_rank_local == \
|
||||
ReconfigureRankType.KEEP_CURRENT_RANK
|
||||
parallel_config.data_parallel_master_ip = \
|
||||
reconfig_request.new_data_parallel_master_ip
|
||||
parallel_config.data_parallel_master_port = \
|
||||
reconfig_request.new_data_parallel_master_port
|
||||
if reconfig_request.new_data_parallel_rank != -2:
|
||||
self.dp_rank = parallel_config.data_parallel_rank
|
||||
self.dp_group = parallel_config.stateless_init_dp_group()
|
||||
reconfig_request.new_data_parallel_master_port = \
|
||||
parallel_config.data_parallel_master_port
|
||||
|
||||
self.model_executor.reinitialize_distributed(reconfig_request)
|
||||
if reconfig_request.new_data_parallel_size > old_dp_size:
|
||||
assert self.available_gpu_memory_for_kv_cache > 0
|
||||
# pass available_gpu_memory_for_kv_cache from existing
|
||||
# engine-cores to new engine-cores so they can directly
|
||||
# use it in _initialize_kv_caches() rather than profiling.
|
||||
ParallelConfig.sync_kv_cache_memory_size(
|
||||
self.dp_group, self.available_gpu_memory_for_kv_cache)
|
||||
# NOTE(yongji): newly joined workers require dummy_run even
|
||||
# CUDA graph is not used
|
||||
self.model_executor.collective_rpc("compile_or_warm_up_model")
|
||||
if reconfig_request.new_data_parallel_rank == \
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK:
|
||||
self.shutdown()
|
||||
logger.info("DPEngineCoreProc %s shutdown", self.dp_rank)
|
||||
else:
|
||||
logger.info("Distributed environment reinitialized for DP rank %s",
|
||||
self.dp_rank)
|
||||
|
||||
|
||||
class DPEngineCoreActor(DPEngineCoreProc):
|
||||
"""
|
||||
|
||||
@@ -21,9 +21,11 @@ import zmq.asyncio
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.utils import get_open_zmq_inproc_path, make_zmq_socket
|
||||
from vllm.utils import get_open_port, get_open_zmq_inproc_path, make_zmq_socket
|
||||
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
EngineCoreRequestType, UtilityOutput)
|
||||
EngineCoreRequestType,
|
||||
ReconfigureDistributedRequest, ReconfigureRankType,
|
||||
UtilityOutput)
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
from vllm.v1.engine.core import EngineCore, EngineCoreProc
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
@@ -162,6 +164,9 @@ class EngineCoreClient(ABC):
|
||||
running state."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -910,14 +915,30 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
events = await poller.poll()
|
||||
if not self.engines_running and len(events) == 2 or (
|
||||
events[0][0] == first_req_rcv_socket):
|
||||
# Send a message to notify the coordinator that
|
||||
# Check if this is a regular request notification or
|
||||
# scale up notification
|
||||
buf = first_req_rcv_socket.recv(
|
||||
flags=zmq.NOBLOCK).result()
|
||||
|
||||
decoded = msgspec.msgpack.decode(buf)
|
||||
if isinstance(
|
||||
decoded,
|
||||
(list, tuple)) and len(decoded) == 2 and decoded[
|
||||
0] == "SCALE_ELASTIC_EP":
|
||||
# Extract new engine count from the decoded message
|
||||
new_engine_count = decoded[1]
|
||||
# Send scale up notification to coordinator
|
||||
scale_msg = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_engine_count))
|
||||
await socket.send(scale_msg)
|
||||
continue
|
||||
|
||||
# we're sending a request while the engines are
|
||||
# paused, so that it can wake the others up
|
||||
# (to run dummy EP loop).
|
||||
assert decoded[0] == "FIRST_REQ"
|
||||
target_eng_index = decoded[1]
|
||||
self.engines_running = True
|
||||
buf = first_req_rcv_socket.recv(
|
||||
flags=zmq.NOBLOCK).result()
|
||||
target_eng_index = int.from_bytes(buf, "little")
|
||||
msg = msgspec.msgpack.encode(
|
||||
(target_eng_index, self.current_wave))
|
||||
await socket.send(msg)
|
||||
@@ -953,7 +974,8 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
chosen_engine)
|
||||
if not self.engines_running:
|
||||
# Notify coordinator that we're sending a request
|
||||
await self.first_req_send_socket.send(chosen_engine)
|
||||
req_msg = msgspec.msgpack.encode(("FIRST_REQ", chosen_engine))
|
||||
await self.first_req_send_socket.send(req_msg)
|
||||
|
||||
await to_await
|
||||
|
||||
@@ -1047,3 +1069,156 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
engine: EngineIdentity) -> None:
|
||||
await self._send_input(EngineCoreRequestType.ABORT, request_ids,
|
||||
engine)
|
||||
|
||||
async def _send_reconfig_message(
|
||||
self, reconfig_request: ReconfigureDistributedRequest,
|
||||
engine: EngineIdentity) -> asyncio.Future:
|
||||
"""Send reconfiguration message and return the result future without
|
||||
waiting for completion."""
|
||||
call_id = uuid.uuid1().int >> 64
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.utility_results[call_id] = future
|
||||
message = (EngineCoreRequestType.UTILITY.value, *self.encoder.encode(
|
||||
(self.client_index, call_id, "reinitialize_distributed",
|
||||
(reconfig_request, ))))
|
||||
await self._send_input_message(message, engine, reconfig_request)
|
||||
self._ensure_output_queue_task()
|
||||
return future
|
||||
|
||||
async def scale_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale elastic EP data parallel size"""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
assert new_data_parallel_size != cur_data_parallel_size, (
|
||||
f"new_data_parallel_size {new_data_parallel_size} must be "
|
||||
f"different from cur_data_parallel_size {cur_data_parallel_size}")
|
||||
|
||||
assert self.vllm_config.parallel_config.data_parallel_backend == \
|
||||
"ray", ("Only ray DP backend supports scaling elastic EP")
|
||||
|
||||
scale_up = new_data_parallel_size > cur_data_parallel_size
|
||||
|
||||
if scale_up:
|
||||
await self._scale_up_elastic_ep(cur_data_parallel_size,
|
||||
new_data_parallel_size)
|
||||
else:
|
||||
await self._scale_down_elastic_ep(cur_data_parallel_size,
|
||||
new_data_parallel_size)
|
||||
|
||||
async def _scale_up_elastic_ep(self, cur_data_parallel_size: int,
|
||||
new_data_parallel_size: int) -> None:
|
||||
"""Scale up the data parallel size by creating new engine cores
|
||||
and reconfiguring existing ones."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
# Phase 1: Send reconfigure messages to all existing engines and wait
|
||||
# for them to be sent
|
||||
reconfig_futures = []
|
||||
self.vllm_config.parallel_config.data_parallel_master_port = \
|
||||
get_open_port()
|
||||
for engine in self.core_engines:
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=\
|
||||
ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=self.vllm_config.parallel_config.
|
||||
data_parallel_master_ip,
|
||||
new_data_parallel_master_port=self.vllm_config.parallel_config.
|
||||
data_parallel_master_port)
|
||||
future = await self._send_reconfig_message(reconfig_request,
|
||||
engine)
|
||||
reconfig_futures.append(future)
|
||||
|
||||
logger.info("All reconfigure messages sent, starting engine creation")
|
||||
|
||||
# Phase 2: Create new engines now that reconfig messages have been sent
|
||||
# self.resources.engine_manager is guaranteed to be
|
||||
# CoreEngineActorManager for RayDPClient
|
||||
assert isinstance(self.resources.engine_manager,
|
||||
CoreEngineActorManager)
|
||||
self.resources.engine_manager.scale_up_elastic_ep(
|
||||
self.vllm_config, new_data_parallel_size)
|
||||
|
||||
# Create new CoreEngine objects for the new engines
|
||||
new_engine_identities = set()
|
||||
for i in range(cur_data_parallel_size, new_data_parallel_size):
|
||||
new_engine = i.to_bytes(2, "little")
|
||||
self.core_engines.append(new_engine)
|
||||
new_engine_identities.add(new_engine)
|
||||
|
||||
# Wait for ready messages from new engines on the input socket
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while new_engine_identities:
|
||||
if not sync_input_socket.poll(timeout=600_000):
|
||||
raise TimeoutError(
|
||||
"Timed out waiting for new engines to send initial "
|
||||
"message on input socket.")
|
||||
identity, _ = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
|
||||
# Phase 3: Wait for all existing engines to complete reconfiguration
|
||||
logger.info("Waiting for existing engines to complete reconfiguration")
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
|
||||
# Notify coordinator about scale up through existing
|
||||
# stats_update_task connection
|
||||
self._ensure_stats_update_task()
|
||||
scale_up_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size))
|
||||
await self.first_req_send_socket.send(scale_up_marker)
|
||||
|
||||
# Update the parallel config
|
||||
self.vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
logger.info(
|
||||
"[Elastic EP] Scale up completed, new data parallel size: %s",
|
||||
new_data_parallel_size)
|
||||
|
||||
async def _scale_down_elastic_ep(self, cur_data_parallel_size: int,
|
||||
new_data_parallel_size: int) -> None:
|
||||
"""Scale down the data parallel size by shutting down and
|
||||
reconfiguring existing engine cores."""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
self.vllm_config.parallel_config.data_parallel_master_port = \
|
||||
get_open_port()
|
||||
|
||||
reconfig_futures = []
|
||||
for cur_dp_rank, engine in enumerate(self.core_engines):
|
||||
reconfig_request = ReconfigureDistributedRequest(
|
||||
new_data_parallel_size=new_data_parallel_size,
|
||||
new_data_parallel_rank=ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_rank_local=\
|
||||
ReconfigureRankType.KEEP_CURRENT_RANK,
|
||||
new_data_parallel_master_ip=self.vllm_config.parallel_config.
|
||||
data_parallel_master_ip,
|
||||
new_data_parallel_master_port=self.vllm_config.parallel_config.
|
||||
data_parallel_master_port)
|
||||
if cur_dp_rank >= new_data_parallel_size:
|
||||
reconfig_request.new_data_parallel_rank = \
|
||||
ReconfigureRankType.SHUTDOWN_CURRENT_RANK
|
||||
future = await self._send_reconfig_message(reconfig_request,
|
||||
engine)
|
||||
reconfig_futures.append(future)
|
||||
|
||||
for _ in range(new_data_parallel_size, cur_data_parallel_size):
|
||||
self.core_engines.pop()
|
||||
|
||||
await asyncio.gather(*reconfig_futures)
|
||||
|
||||
assert isinstance(self.resources.engine_manager,
|
||||
CoreEngineActorManager)
|
||||
self.resources.engine_manager.scale_down_elastic_ep(
|
||||
cur_data_parallel_size, new_data_parallel_size)
|
||||
|
||||
self._ensure_stats_update_task()
|
||||
scale_down_marker = msgspec.msgpack.encode(
|
||||
("SCALE_ELASTIC_EP", new_data_parallel_size))
|
||||
await self.first_req_send_socket.send(scale_down_marker)
|
||||
|
||||
self.vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
logger.info(
|
||||
"[Elastic EP] Scale down completed, new data parallel size: %s",
|
||||
new_data_parallel_size)
|
||||
|
||||
+217
-8
@@ -174,16 +174,21 @@ class CoreEngineActorManager:
|
||||
|
||||
self.local_engine_actors: list[ray.ActorHandle] = []
|
||||
self.remote_engine_actors: list[ray.ActorHandle] = []
|
||||
|
||||
env_vars_list = get_env_vars_to_copy(destination="DPEngineCoreActor")
|
||||
self.env_vars_dict = {
|
||||
name: os.environ[name]
|
||||
for name in env_vars_list if name in os.environ
|
||||
}
|
||||
runtime_env = RuntimeEnv(env_vars=self.env_vars_dict)
|
||||
|
||||
self.addresses = addresses
|
||||
self.executor_class = executor_class
|
||||
self.log_stats = log_stats
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
local_engine_count = \
|
||||
vllm_config.parallel_config.data_parallel_size_local
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
env_vars_set = get_env_vars_to_copy(destination="DPEngineCoreActor")
|
||||
env_vars_dict = {
|
||||
name: os.environ[name]
|
||||
for name in env_vars_set if name in os.environ
|
||||
}
|
||||
runtime_env = RuntimeEnv(env_vars=env_vars_dict)
|
||||
|
||||
if ray.is_initialized():
|
||||
logger.info(
|
||||
@@ -208,6 +213,7 @@ class CoreEngineActorManager:
|
||||
assert len(placement_groups) == dp_size, (
|
||||
"Number of placement groups must match data parallel size")
|
||||
|
||||
self.placement_group_is_local = []
|
||||
refs = []
|
||||
for index in range(dp_size):
|
||||
local_index = local_dp_ranks[index]
|
||||
@@ -231,6 +237,7 @@ class CoreEngineActorManager:
|
||||
self.local_engine_actors.append(actor)
|
||||
else:
|
||||
self.remote_engine_actors.append(actor)
|
||||
self.placement_group_is_local.append(local_client)
|
||||
refs.append(actor.wait_for_init.remote())
|
||||
|
||||
ray.get(refs)
|
||||
@@ -242,6 +249,9 @@ class CoreEngineActorManager:
|
||||
def create_dp_placement_groups(
|
||||
vllm_config: VllmConfig
|
||||
) -> tuple[list["PlacementGroup"], list[int]]:
|
||||
"""
|
||||
Create placement groups for data parallel.
|
||||
"""
|
||||
|
||||
import ray
|
||||
from ray._private.state import available_resources_per_node
|
||||
@@ -250,10 +260,11 @@ class CoreEngineActorManager:
|
||||
logger.info("Creating placement groups for data parallel")
|
||||
dp_master_ip = \
|
||||
vllm_config.parallel_config.data_parallel_master_ip
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
num_pg_to_create = vllm_config.parallel_config.data_parallel_size
|
||||
local_engine_count = \
|
||||
vllm_config.parallel_config.data_parallel_size_local
|
||||
|
||||
nodes = list_nodes()
|
||||
nodes = sorted(list_nodes(),
|
||||
key=lambda node: node.node_ip != dp_master_ip)
|
||||
assert nodes[0].node_ip == dp_master_ip, (
|
||||
@@ -293,7 +304,7 @@ class CoreEngineActorManager:
|
||||
local_dp_ranks.append(i)
|
||||
else:
|
||||
for i in range(available_engine_count):
|
||||
if len(placement_groups) == dp_size:
|
||||
if len(placement_groups) == num_pg_to_create:
|
||||
break
|
||||
bundles = [{"GPU": 1.0}] * world_size + [{"CPU": 1.0}]
|
||||
pg = ray.util.placement_group(
|
||||
@@ -305,6 +316,204 @@ class CoreEngineActorManager:
|
||||
local_dp_ranks.append(i)
|
||||
return placement_groups, local_dp_ranks
|
||||
|
||||
@staticmethod
|
||||
def add_dp_placement_groups(
|
||||
old_vllm_config: VllmConfig, new_data_parallel_size: int
|
||||
) -> tuple[list["PlacementGroup"], list[int]]:
|
||||
"""
|
||||
Add placement groups for new data parallel size.
|
||||
"""
|
||||
import ray
|
||||
from ray._private.state import (available_resources_per_node,
|
||||
total_resources_per_node)
|
||||
from ray.util.state import list_nodes
|
||||
|
||||
old_dp_size = old_vllm_config.parallel_config.data_parallel_size
|
||||
num_pg_to_create = new_data_parallel_size - old_dp_size
|
||||
|
||||
if num_pg_to_create <= 0:
|
||||
return [], []
|
||||
|
||||
dp_master_ip = old_vllm_config.parallel_config.data_parallel_master_ip
|
||||
world_size = old_vllm_config.parallel_config.world_size
|
||||
|
||||
nodes = list_nodes()
|
||||
nodes = sorted(nodes, key=lambda node: node.node_ip != dp_master_ip)
|
||||
assert nodes[0].node_ip == dp_master_ip, (
|
||||
"The first node must be the head node")
|
||||
assert len(nodes) == 1 or nodes[1].node_ip != dp_master_ip, (
|
||||
"There can only be one head node")
|
||||
|
||||
available_resources = available_resources_per_node()
|
||||
total_resources = total_resources_per_node()
|
||||
|
||||
placement_groups = []
|
||||
local_dp_ranks = []
|
||||
num_pg_created = 0
|
||||
|
||||
for node in nodes:
|
||||
if num_pg_created >= num_pg_to_create:
|
||||
break
|
||||
|
||||
node_ip = node.node_ip
|
||||
node_id = node.node_id
|
||||
available_gpus = int(available_resources[node_id]["GPU"])
|
||||
|
||||
# Get total GPUs on this node from the node's resources
|
||||
# Ray stores node resources with node ID as key
|
||||
total_gpus = int(total_resources[node_id]["GPU"])
|
||||
|
||||
# Calculate used GPUs and used engines on this node
|
||||
used_gpus = max(0, total_gpus - available_gpus)
|
||||
used_engines_on_node = used_gpus // world_size
|
||||
|
||||
# Calculate how many new engines this node can accommodate
|
||||
available_engine_count = available_gpus // world_size
|
||||
|
||||
# Create placement groups for new engines on this node
|
||||
for i in range(available_engine_count):
|
||||
if num_pg_created >= num_pg_to_create:
|
||||
break
|
||||
|
||||
rank = old_dp_size + num_pg_created
|
||||
|
||||
# Create bundles with node constraint for master node
|
||||
if node_ip == dp_master_ip:
|
||||
bundles = [{
|
||||
"GPU": 1.0,
|
||||
"node:" + dp_master_ip: 0.001
|
||||
}] * world_size + [{
|
||||
"CPU": 1.0
|
||||
}]
|
||||
else:
|
||||
bundles = [{"GPU": 1.0}] * world_size + [{"CPU": 1.0}]
|
||||
|
||||
pg = ray.util.placement_group(
|
||||
name=f"dp_rank_{rank}",
|
||||
strategy="STRICT_PACK",
|
||||
bundles=bundles,
|
||||
)
|
||||
placement_groups.append(pg)
|
||||
|
||||
# Local rank starts from the number of engines already used
|
||||
# on this node
|
||||
local_rank = used_engines_on_node + i
|
||||
local_dp_ranks.append(local_rank)
|
||||
num_pg_created += 1
|
||||
|
||||
return placement_groups, local_dp_ranks
|
||||
|
||||
def scale_up_elastic_ep(self, cur_vllm_config: VllmConfig,
|
||||
new_data_parallel_size: int) -> None:
|
||||
import copy
|
||||
|
||||
import ray
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
from ray.util.scheduling_strategies import (
|
||||
PlacementGroupSchedulingStrategy)
|
||||
|
||||
from vllm.v1.engine.core import DPEngineCoreActor
|
||||
|
||||
cur_data_parallel_size = len(self.local_engine_actors) + \
|
||||
len(self.remote_engine_actors)
|
||||
|
||||
assert new_data_parallel_size > cur_data_parallel_size, (
|
||||
f"New data parallel size {new_data_parallel_size} must be greater "
|
||||
f"than current data parallel size {cur_data_parallel_size} "
|
||||
"for scale up")
|
||||
|
||||
placement_groups, local_dp_ranks = \
|
||||
self.add_dp_placement_groups(
|
||||
cur_vllm_config, new_data_parallel_size)
|
||||
|
||||
world_size = cur_vllm_config.parallel_config.world_size
|
||||
dp_master_ip = cur_vllm_config.parallel_config.data_parallel_master_ip
|
||||
new_local_engines = 0
|
||||
|
||||
runtime_env = RuntimeEnv(env_vars=self.env_vars_dict
|
||||
| {"VLLM_ELASTIC_EP_SCALE_UP_LAUNCH": "1"})
|
||||
for i, (pg,
|
||||
local_rank) in enumerate(zip(placement_groups,
|
||||
local_dp_ranks)):
|
||||
rank = cur_data_parallel_size + i
|
||||
dp_vllm_config = copy.deepcopy(cur_vllm_config)
|
||||
dp_vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
dp_vllm_config.parallel_config.placement_group = pg
|
||||
|
||||
# Check if this placement group is on the head node
|
||||
local_client = any(
|
||||
bundle.get("node:" + dp_master_ip, 0) > 0
|
||||
for bundle in pg.bundle_specs)
|
||||
|
||||
if local_client:
|
||||
new_local_engines += 1
|
||||
# Update data_parallel_size_local
|
||||
dp_vllm_config.parallel_config.data_parallel_size_local = (
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local +
|
||||
new_local_engines)
|
||||
|
||||
actor = ray.remote(DPEngineCoreActor).options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
placement_group_bundle_index=world_size,
|
||||
),
|
||||
runtime_env=runtime_env).remote(
|
||||
vllm_config=dp_vllm_config,
|
||||
executor_class=self.executor_class,
|
||||
log_stats=self.log_stats,
|
||||
local_client=local_client,
|
||||
addresses=self.addresses,
|
||||
dp_rank=rank,
|
||||
local_dp_rank=local_rank)
|
||||
|
||||
if local_client:
|
||||
self.local_engine_actors.append(actor)
|
||||
else:
|
||||
self.remote_engine_actors.append(actor)
|
||||
self.created_placement_groups.append(pg)
|
||||
self.placement_group_is_local.append(local_client)
|
||||
|
||||
ray.get([
|
||||
actor.wait_for_init.remote()
|
||||
for actor in (self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0 else []) +
|
||||
self.remote_engine_actors[-(len(placement_groups) -
|
||||
new_local_engines):]
|
||||
])
|
||||
|
||||
actors = (self.local_engine_actors[-new_local_engines:]
|
||||
if new_local_engines > 0 else []) + \
|
||||
self.remote_engine_actors[-(len(placement_groups) -
|
||||
new_local_engines):]
|
||||
|
||||
for actor in actors:
|
||||
self.run_refs.append(actor.run.remote())
|
||||
|
||||
cur_vllm_config.parallel_config.data_parallel_size = \
|
||||
new_data_parallel_size
|
||||
# Update old_vllm_config with new data_parallel_size_local if any new
|
||||
# local engines were added
|
||||
if new_local_engines > 0:
|
||||
cur_vllm_config.parallel_config.data_parallel_size_local += \
|
||||
new_local_engines
|
||||
|
||||
def scale_down_elastic_ep(self, cur_data_parallel_size: int,
|
||||
new_data_parallel_size: int) -> None:
|
||||
import ray
|
||||
assert cur_data_parallel_size > new_data_parallel_size, (
|
||||
f"cur_data_parallel_size {cur_data_parallel_size} must be greater "
|
||||
f"than new_data_parallel_size {new_data_parallel_size} "
|
||||
"for scale down")
|
||||
for _ in range(cur_data_parallel_size - new_data_parallel_size):
|
||||
pg = self.created_placement_groups.pop()
|
||||
is_local = self.placement_group_is_local.pop()
|
||||
if is_local:
|
||||
self.local_engine_actors.pop()
|
||||
else:
|
||||
self.remote_engine_actors.pop()
|
||||
ray.util.remove_placement_group(pg)
|
||||
|
||||
def get_run_refs(self):
|
||||
return self.run_refs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user