[BugFix] Support online dense model DP without overhead (#30739)

Signed-off-by: Nick Hill <nhill@redhat.com>
Signed-off-by: njhill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-01-02 23:36:38 +08:00
committed by GitHub
parent 08f425bad1
commit bd877162eb
20 changed files with 345 additions and 146 deletions
+96 -25
View File
@@ -84,6 +84,7 @@ class EngineCore:
executor_class: type[Executor],
log_stats: bool,
executor_fail_callback: Callable | None = None,
include_finished_set: bool = False,
):
# plugins need to be loaded at the engine/scheduler level too
from vllm.plugins import load_general_plugins
@@ -91,7 +92,7 @@ class EngineCore:
load_general_plugins()
self.vllm_config = vllm_config
if vllm_config.parallel_config.data_parallel_rank == 0:
if not vllm_config.parallel_config.data_parallel_rank_local:
logger.info(
"Initializing a V1 LLM engine (v%s) with config: %s",
VLLM_VERSION,
@@ -138,7 +139,7 @@ class EngineCore:
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
structured_output_manager=self.structured_output_manager,
include_finished_set=vllm_config.parallel_config.data_parallel_size > 1,
include_finished_set=include_finished_set,
log_stats=self.log_stats,
block_size=scheduler_block_size,
)
@@ -605,6 +606,7 @@ class EngineCoreProc(EngineCore):
executor_class: type[Executor],
log_stats: bool,
client_handshake_address: str | None = None,
*,
engine_index: int = 0,
):
self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]()
@@ -636,17 +638,22 @@ class EngineCoreProc(EngineCore):
self.has_coordinator,
self.frontend_stats_publish_address,
)
# Only publish request queue stats to coordinator for "internal"
# and "hybrid" LB modes .
self.publish_dp_lb_stats = (
internal_dp_balancing = (
self.has_coordinator
and not vllm_config.parallel_config.data_parallel_external_lb
)
# Only publish request queue stats to coordinator for "internal"
# and "hybrid" LB modes.
self.publish_dp_lb_stats = internal_dp_balancing
self._init_data_parallel(vllm_config)
super().__init__(
vllm_config, executor_class, log_stats, executor_fail_callback
vllm_config,
executor_class,
log_stats,
executor_fail_callback,
internal_dp_balancing,
)
# Background Threads and Queues for IO. These enable us to
@@ -854,18 +861,29 @@ class EngineCoreProc(EngineCore):
engine_core: EngineCoreProc | None = None
try:
parallel_config: ParallelConfig = kwargs["vllm_config"].parallel_config
if parallel_config.data_parallel_size > 1 or dp_rank > 0:
set_process_title("EngineCore", f"DP{dp_rank}")
decorate_logs()
# Set data parallel rank for this engine process.
parallel_config.data_parallel_rank = dp_rank
vllm_config: VllmConfig = kwargs["vllm_config"]
parallel_config: ParallelConfig = vllm_config.parallel_config
data_parallel = parallel_config.data_parallel_size > 1 or dp_rank > 0
if data_parallel:
parallel_config.data_parallel_rank_local = local_dp_rank
engine_core = DPEngineCoreProc(*args, **kwargs)
set_process_title("EngineCore", f"DP{dp_rank}")
else:
set_process_title("EngineCore")
decorate_logs()
engine_core = EngineCoreProc(*args, **kwargs)
decorate_logs()
parallel_config.data_parallel_index = dp_rank
if data_parallel and vllm_config.model_config.is_moe:
# Set data parallel rank for this engine process.
parallel_config.data_parallel_rank = dp_rank
engine_core = DPEngineCoreProc(*args, **kwargs)
else:
# Non-MoE DP ranks are completely independent, so treat like DP=1.
# Note that parallel_config.data_parallel_index will still reflect
# the original DP rank.
parallel_config.data_parallel_size = 1
parallel_config.data_parallel_size_local = 1
parallel_config.data_parallel_rank = 0
engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
engine_core.run_busy_loop()
@@ -1195,6 +1213,10 @@ class DPEngineCoreProc(EngineCoreProc):
log_stats: bool,
client_handshake_address: str | None = None,
):
assert vllm_config.model_config.is_moe, (
"DPEngineCoreProc should only be used for MoE models"
)
# Counts forward-passes of the model so that we can synchronize
# finished with DP peers every N steps.
self.step_counter = 0
@@ -1210,7 +1232,7 @@ class DPEngineCoreProc(EngineCoreProc):
executor_class,
log_stats,
client_handshake_address,
dp_rank,
engine_index=dp_rank,
)
def _init_data_parallel(self, vllm_config: VllmConfig):
@@ -1391,7 +1413,7 @@ class DPEngineCoreProc(EngineCoreProc):
)
class DPEngineCoreActor(DPEngineCoreProc):
class EngineCoreActorMixin:
"""
Ray actor for running EngineCore in a data parallel context
"""
@@ -1399,15 +1421,12 @@ class DPEngineCoreActor(DPEngineCoreProc):
def __init__(
self,
vllm_config: VllmConfig,
local_client: bool,
addresses: EngineZmqAddresses,
executor_class: type[Executor],
log_stats: bool,
dp_rank: int = 0,
local_dp_rank: int = 0,
):
self.addresses = addresses
vllm_config.parallel_config.data_parallel_rank = dp_rank
vllm_config.parallel_config.data_parallel_index = dp_rank
vllm_config.parallel_config.data_parallel_rank_local = local_dp_rank
# Set CUDA_VISIBLE_DEVICES as early as possible in actor life cycle
@@ -1429,8 +1448,6 @@ class DPEngineCoreActor(DPEngineCoreProc):
# of ray.
self._set_visible_devices(vllm_config, local_dp_rank)
super().__init__(vllm_config, local_client, "", executor_class, log_stats)
def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int):
from vllm.platforms import current_platform
@@ -1491,7 +1508,7 @@ class DPEngineCoreActor(DPEngineCoreProc):
Run the engine core busy loop.
"""
try:
self.run_busy_loop()
self.run_busy_loop() # type: ignore[attr-defined]
except SystemExit:
logger.debug("EngineCore exiting.")
raise
@@ -1499,4 +1516,58 @@ class DPEngineCoreActor(DPEngineCoreProc):
logger.exception("EngineCore encountered a fatal error.")
raise
finally:
self.shutdown()
self.shutdown() # type: ignore[attr-defined]
class DPMoEEngineCoreActor(EngineCoreActorMixin, DPEngineCoreProc):
"""Used for MoE model data parallel cases."""
def __init__(
self,
vllm_config: VllmConfig,
local_client: bool,
addresses: EngineZmqAddresses,
executor_class: type[Executor],
log_stats: bool,
dp_rank: int = 0,
local_dp_rank: int = 0,
):
vllm_config.parallel_config.data_parallel_rank = dp_rank
EngineCoreActorMixin.__init__(
self, vllm_config, addresses, dp_rank, local_dp_rank
)
DPEngineCoreProc.__init__(
self, vllm_config, local_client, "", executor_class, log_stats
)
class EngineCoreActor(EngineCoreActorMixin, EngineCoreProc):
"""Used for non-MoE and/or non-DP cases."""
def __init__(
self,
vllm_config: VllmConfig,
local_client: bool,
addresses: EngineZmqAddresses,
executor_class: type[Executor],
log_stats: bool,
dp_rank: int = 0,
local_dp_rank: int = 0,
):
vllm_config.parallel_config.data_parallel_size = 1
vllm_config.parallel_config.data_parallel_size_local = 1
vllm_config.parallel_config.data_parallel_rank = 0
EngineCoreActorMixin.__init__(
self, vllm_config, addresses, dp_rank, local_dp_rank
)
EngineCoreProc.__init__(
self,
vllm_config,
local_client,
"",
executor_class,
log_stats,
engine_index=dp_rank,
)