[BugFix] Support online dense model DP without overhead (#30739)
Signed-off-by: Nick Hill <nhill@redhat.com> Signed-off-by: njhill <nickhill123@gmail.com>
This commit is contained in:
@@ -55,7 +55,9 @@ class DPCoordinator:
|
||||
request wave / running state changes.
|
||||
"""
|
||||
|
||||
def __init__(self, parallel_config: ParallelConfig):
|
||||
def __init__(
|
||||
self, parallel_config: ParallelConfig, enable_wave_coordination: bool = True
|
||||
):
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
assert dp_size > 1, "Coordinator only used for data parallel"
|
||||
|
||||
@@ -83,6 +85,7 @@ class DPCoordinator:
|
||||
"front_publish_address": front_publish_address,
|
||||
"back_output_address": back_output_address,
|
||||
"back_publish_address": back_publish_address,
|
||||
"enable_wave_coordination": enable_wave_coordination,
|
||||
},
|
||||
daemon=True,
|
||||
)
|
||||
@@ -110,13 +113,19 @@ class EngineState:
|
||||
|
||||
|
||||
class DPCoordinatorProc:
|
||||
def __init__(self, engine_count: int, min_stats_update_interval_ms: int = 100):
|
||||
def __init__(
|
||||
self,
|
||||
engine_count: int,
|
||||
min_stats_update_interval_ms: int = 100,
|
||||
enable_wave_coordination: bool = True,
|
||||
):
|
||||
set_process_title("DPCoordinator")
|
||||
self.ctx = zmq.Context()
|
||||
|
||||
self.engines = [EngineState() for _ in range(engine_count)]
|
||||
|
||||
self.stats_update_interval_ms = min_stats_update_interval_ms
|
||||
self.enable_wave_coordination = enable_wave_coordination
|
||||
|
||||
@staticmethod
|
||||
def run_coordinator(
|
||||
@@ -125,10 +134,12 @@ class DPCoordinatorProc:
|
||||
back_output_address: str,
|
||||
back_publish_address: str,
|
||||
min_stats_update_interval_ms: int = 100,
|
||||
enable_wave_coordination: bool = True,
|
||||
):
|
||||
coordinator = DPCoordinatorProc(
|
||||
engine_count=engine_count,
|
||||
min_stats_update_interval_ms=min_stats_update_interval_ms,
|
||||
enable_wave_coordination=enable_wave_coordination,
|
||||
)
|
||||
try:
|
||||
coordinator.process_input_socket(
|
||||
@@ -265,22 +276,25 @@ class DPCoordinatorProc:
|
||||
)
|
||||
continue # Skip normal engine notification processing
|
||||
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
# Wave coordination: handle new-request messages from front-end.
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = decoded
|
||||
if not engines_running:
|
||||
if wave < current_wave:
|
||||
# If the wave number is stale, ensure the message
|
||||
# is handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
self._send_start_wave(
|
||||
publish_back, current_wave, engine_to_exclude
|
||||
)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
@@ -325,34 +339,39 @@ class DPCoordinatorProc:
|
||||
stats[1] = scheduler_stats.num_running_reqs
|
||||
stats_changed = True
|
||||
|
||||
if (wave := outputs.wave_complete) is not None:
|
||||
# 2. Notification from rank 0 engine that we've
|
||||
# moved into the global paused state
|
||||
# (engines_running==False).
|
||||
if current_wave <= wave:
|
||||
new_wave = wave + 1
|
||||
# Wave coordination: handle wave completion and start notifications
|
||||
# Only process these when wave coordination is enabled
|
||||
if self.enable_wave_coordination:
|
||||
if (wave := outputs.wave_complete) is not None:
|
||||
# 2. Notification from rank 0 engine that we've
|
||||
# moved into the global paused state
|
||||
# (engines_running==False).
|
||||
if current_wave <= wave:
|
||||
new_wave = wave + 1
|
||||
logger.debug(
|
||||
"Moving DP wave from %d to %d.",
|
||||
current_wave,
|
||||
new_wave,
|
||||
)
|
||||
current_wave = new_wave
|
||||
engines_running = False
|
||||
wave_state_changed = True
|
||||
elif (wave := outputs.start_wave) is not None and (
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
logger.debug(
|
||||
"Moving DP wave from %d to %d.", current_wave, new_wave
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
wave,
|
||||
)
|
||||
current_wave = new_wave
|
||||
engines_running = False
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
elif (wave := outputs.start_wave) is not None and (
|
||||
wave > current_wave
|
||||
or (wave == current_wave and not engines_running)
|
||||
):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.",
|
||||
wave,
|
||||
)
|
||||
current_wave = wave
|
||||
engines_running = True
|
||||
wave_state_changed = True
|
||||
self._send_start_wave(publish_back, wave, eng_index)
|
||||
self._send_start_wave(publish_back, wave, eng_index)
|
||||
|
||||
if wave_state_changed:
|
||||
message = (None, current_wave, engines_running)
|
||||
|
||||
+96
-25
@@ -84,6 +84,7 @@ class EngineCore:
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
executor_fail_callback: Callable | None = None,
|
||||
include_finished_set: bool = False,
|
||||
):
|
||||
# plugins need to be loaded at the engine/scheduler level too
|
||||
from vllm.plugins import load_general_plugins
|
||||
@@ -91,7 +92,7 @@ class EngineCore:
|
||||
load_general_plugins()
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
if vllm_config.parallel_config.data_parallel_rank == 0:
|
||||
if not vllm_config.parallel_config.data_parallel_rank_local:
|
||||
logger.info(
|
||||
"Initializing a V1 LLM engine (v%s) with config: %s",
|
||||
VLLM_VERSION,
|
||||
@@ -138,7 +139,7 @@ class EngineCore:
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
structured_output_manager=self.structured_output_manager,
|
||||
include_finished_set=vllm_config.parallel_config.data_parallel_size > 1,
|
||||
include_finished_set=include_finished_set,
|
||||
log_stats=self.log_stats,
|
||||
block_size=scheduler_block_size,
|
||||
)
|
||||
@@ -605,6 +606,7 @@ class EngineCoreProc(EngineCore):
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_handshake_address: str | None = None,
|
||||
*,
|
||||
engine_index: int = 0,
|
||||
):
|
||||
self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]()
|
||||
@@ -636,17 +638,22 @@ class EngineCoreProc(EngineCore):
|
||||
self.has_coordinator,
|
||||
self.frontend_stats_publish_address,
|
||||
)
|
||||
# Only publish request queue stats to coordinator for "internal"
|
||||
# and "hybrid" LB modes .
|
||||
self.publish_dp_lb_stats = (
|
||||
internal_dp_balancing = (
|
||||
self.has_coordinator
|
||||
and not vllm_config.parallel_config.data_parallel_external_lb
|
||||
)
|
||||
# Only publish request queue stats to coordinator for "internal"
|
||||
# and "hybrid" LB modes.
|
||||
self.publish_dp_lb_stats = internal_dp_balancing
|
||||
|
||||
self._init_data_parallel(vllm_config)
|
||||
|
||||
super().__init__(
|
||||
vllm_config, executor_class, log_stats, executor_fail_callback
|
||||
vllm_config,
|
||||
executor_class,
|
||||
log_stats,
|
||||
executor_fail_callback,
|
||||
internal_dp_balancing,
|
||||
)
|
||||
|
||||
# Background Threads and Queues for IO. These enable us to
|
||||
@@ -854,18 +861,29 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
engine_core: EngineCoreProc | None = None
|
||||
try:
|
||||
parallel_config: ParallelConfig = kwargs["vllm_config"].parallel_config
|
||||
if parallel_config.data_parallel_size > 1 or dp_rank > 0:
|
||||
set_process_title("EngineCore", f"DP{dp_rank}")
|
||||
decorate_logs()
|
||||
# Set data parallel rank for this engine process.
|
||||
parallel_config.data_parallel_rank = dp_rank
|
||||
vllm_config: VllmConfig = kwargs["vllm_config"]
|
||||
parallel_config: ParallelConfig = vllm_config.parallel_config
|
||||
data_parallel = parallel_config.data_parallel_size > 1 or dp_rank > 0
|
||||
if data_parallel:
|
||||
parallel_config.data_parallel_rank_local = local_dp_rank
|
||||
engine_core = DPEngineCoreProc(*args, **kwargs)
|
||||
set_process_title("EngineCore", f"DP{dp_rank}")
|
||||
else:
|
||||
set_process_title("EngineCore")
|
||||
decorate_logs()
|
||||
engine_core = EngineCoreProc(*args, **kwargs)
|
||||
decorate_logs()
|
||||
|
||||
parallel_config.data_parallel_index = dp_rank
|
||||
if data_parallel and vllm_config.model_config.is_moe:
|
||||
# Set data parallel rank for this engine process.
|
||||
parallel_config.data_parallel_rank = dp_rank
|
||||
engine_core = DPEngineCoreProc(*args, **kwargs)
|
||||
else:
|
||||
# Non-MoE DP ranks are completely independent, so treat like DP=1.
|
||||
# Note that parallel_config.data_parallel_index will still reflect
|
||||
# the original DP rank.
|
||||
parallel_config.data_parallel_size = 1
|
||||
parallel_config.data_parallel_size_local = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
|
||||
|
||||
engine_core.run_busy_loop()
|
||||
|
||||
@@ -1195,6 +1213,10 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
log_stats: bool,
|
||||
client_handshake_address: str | None = None,
|
||||
):
|
||||
assert vllm_config.model_config.is_moe, (
|
||||
"DPEngineCoreProc should only be used for MoE models"
|
||||
)
|
||||
|
||||
# Counts forward-passes of the model so that we can synchronize
|
||||
# finished with DP peers every N steps.
|
||||
self.step_counter = 0
|
||||
@@ -1210,7 +1232,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
executor_class,
|
||||
log_stats,
|
||||
client_handshake_address,
|
||||
dp_rank,
|
||||
engine_index=dp_rank,
|
||||
)
|
||||
|
||||
def _init_data_parallel(self, vllm_config: VllmConfig):
|
||||
@@ -1391,7 +1413,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
)
|
||||
|
||||
|
||||
class DPEngineCoreActor(DPEngineCoreProc):
|
||||
class EngineCoreActorMixin:
|
||||
"""
|
||||
Ray actor for running EngineCore in a data parallel context
|
||||
"""
|
||||
@@ -1399,15 +1421,12 @@ class DPEngineCoreActor(DPEngineCoreProc):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
local_client: bool,
|
||||
addresses: EngineZmqAddresses,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
dp_rank: int = 0,
|
||||
local_dp_rank: int = 0,
|
||||
):
|
||||
self.addresses = addresses
|
||||
vllm_config.parallel_config.data_parallel_rank = dp_rank
|
||||
vllm_config.parallel_config.data_parallel_index = dp_rank
|
||||
vllm_config.parallel_config.data_parallel_rank_local = local_dp_rank
|
||||
|
||||
# Set CUDA_VISIBLE_DEVICES as early as possible in actor life cycle
|
||||
@@ -1429,8 +1448,6 @@ class DPEngineCoreActor(DPEngineCoreProc):
|
||||
# of ray.
|
||||
self._set_visible_devices(vllm_config, local_dp_rank)
|
||||
|
||||
super().__init__(vllm_config, local_client, "", executor_class, log_stats)
|
||||
|
||||
def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -1491,7 +1508,7 @@ class DPEngineCoreActor(DPEngineCoreProc):
|
||||
Run the engine core busy loop.
|
||||
"""
|
||||
try:
|
||||
self.run_busy_loop()
|
||||
self.run_busy_loop() # type: ignore[attr-defined]
|
||||
except SystemExit:
|
||||
logger.debug("EngineCore exiting.")
|
||||
raise
|
||||
@@ -1499,4 +1516,58 @@ class DPEngineCoreActor(DPEngineCoreProc):
|
||||
logger.exception("EngineCore encountered a fatal error.")
|
||||
raise
|
||||
finally:
|
||||
self.shutdown()
|
||||
self.shutdown() # type: ignore[attr-defined]
|
||||
|
||||
|
||||
class DPMoEEngineCoreActor(EngineCoreActorMixin, DPEngineCoreProc):
|
||||
"""Used for MoE model data parallel cases."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
local_client: bool,
|
||||
addresses: EngineZmqAddresses,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
dp_rank: int = 0,
|
||||
local_dp_rank: int = 0,
|
||||
):
|
||||
vllm_config.parallel_config.data_parallel_rank = dp_rank
|
||||
|
||||
EngineCoreActorMixin.__init__(
|
||||
self, vllm_config, addresses, dp_rank, local_dp_rank
|
||||
)
|
||||
DPEngineCoreProc.__init__(
|
||||
self, vllm_config, local_client, "", executor_class, log_stats
|
||||
)
|
||||
|
||||
|
||||
class EngineCoreActor(EngineCoreActorMixin, EngineCoreProc):
|
||||
"""Used for non-MoE and/or non-DP cases."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
local_client: bool,
|
||||
addresses: EngineZmqAddresses,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
dp_rank: int = 0,
|
||||
local_dp_rank: int = 0,
|
||||
):
|
||||
vllm_config.parallel_config.data_parallel_size = 1
|
||||
vllm_config.parallel_config.data_parallel_size_local = 1
|
||||
vllm_config.parallel_config.data_parallel_rank = 0
|
||||
|
||||
EngineCoreActorMixin.__init__(
|
||||
self, vllm_config, addresses, dp_rank, local_dp_rank
|
||||
)
|
||||
EngineCoreProc.__init__(
|
||||
self,
|
||||
vllm_config,
|
||||
local_client,
|
||||
"",
|
||||
executor_class,
|
||||
log_stats,
|
||||
engine_index=dp_rank,
|
||||
)
|
||||
|
||||
@@ -502,7 +502,7 @@ class MPClient(EngineCoreClient):
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
dp_local_size = parallel_config.data_parallel_size_local
|
||||
offline_mode = parallel_config.data_parallel_rank_local is not None
|
||||
# Client manages local+remote EngineCores in pure internal LB case.
|
||||
|
||||
@@ -65,8 +65,9 @@ class LLMEngine:
|
||||
|
||||
self.log_stats = log_stats
|
||||
|
||||
executor_backend = self.vllm_config.parallel_config.distributed_executor_backend
|
||||
parallel_config = vllm_config.parallel_config
|
||||
executor_backend = parallel_config.distributed_executor_backend
|
||||
|
||||
self.external_launcher_dp = (
|
||||
parallel_config.data_parallel_size > 1
|
||||
and executor_backend == "external_launcher"
|
||||
|
||||
+37
-19
@@ -75,7 +75,6 @@ class EngineHandshakeMetadata:
|
||||
|
||||
addresses: EngineZmqAddresses
|
||||
parallel_config: dict[str, int | str | list[int]]
|
||||
parallel_config_hash: str | None = None
|
||||
|
||||
|
||||
class CoreEngineProcManager:
|
||||
@@ -249,12 +248,19 @@ class CoreEngineActorManager:
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
from vllm.v1.engine.core import DPEngineCoreActor
|
||||
from vllm.v1.engine.core import DPMoEEngineCoreActor, EngineCoreActor
|
||||
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
actor_class = (
|
||||
DPMoEEngineCoreActor
|
||||
if dp_size > 1 and vllm_config.model_config.is_moe
|
||||
else EngineCoreActor
|
||||
)
|
||||
|
||||
self.local_engine_actors: list[ray.ActorHandle] = []
|
||||
self.remote_engine_actors: list[ray.ActorHandle] = []
|
||||
|
||||
env_vars_list = get_env_vars_to_copy(destination="DPEngineCoreActor")
|
||||
env_vars_list = get_env_vars_to_copy(destination=actor_class.__name__)
|
||||
self.env_vars_dict = {
|
||||
name: os.environ[name] for name in env_vars_list if name in os.environ
|
||||
}
|
||||
@@ -263,7 +269,6 @@ class CoreEngineActorManager:
|
||||
self.addresses = addresses
|
||||
self.executor_class = executor_class
|
||||
self.log_stats = log_stats
|
||||
dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
local_engine_count = vllm_config.parallel_config.data_parallel_size_local
|
||||
world_size = vllm_config.parallel_config.world_size
|
||||
|
||||
@@ -314,7 +319,7 @@ class CoreEngineActorManager:
|
||||
runtime_env = RuntimeEnv(env_vars=actor_env_vars)
|
||||
|
||||
actor = (
|
||||
ray.remote(DPEngineCoreActor)
|
||||
ray.remote(actor_class)
|
||||
.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
@@ -624,7 +629,13 @@ class CoreEngineActorManager:
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
from vllm.v1.engine.core import DPEngineCoreActor
|
||||
from vllm.v1.engine.core import DPMoEEngineCoreActor, EngineCoreActor
|
||||
|
||||
actor_class = (
|
||||
DPMoEEngineCoreActor
|
||||
if cur_vllm_config.model_config.is_moe
|
||||
else EngineCoreActor
|
||||
)
|
||||
|
||||
cur_data_parallel_size = len(self.local_engine_actors) + len(
|
||||
self.remote_engine_actors
|
||||
@@ -667,7 +678,7 @@ class CoreEngineActorManager:
|
||||
)
|
||||
|
||||
actor = (
|
||||
ray.remote(DPEngineCoreActor)
|
||||
ray.remote(actor_class)
|
||||
.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
@@ -804,12 +815,19 @@ def launch_core_engines(
|
||||
],
|
||||
)
|
||||
|
||||
# Run the DP Coordinator process with rank 0 when in
|
||||
# online DP mode.
|
||||
run_coordinator = dp_size > 1 and not offline_mode and dp_rank == 0
|
||||
# Run the DP Coordinator process with rank 0 when in online DP mode.
|
||||
# The coordinator is needed for:
|
||||
# 1. Internal/hybrid LB: collecting and publishing queue stats for load balancing
|
||||
# 2. MoE models: wave coordination in addition to stats
|
||||
run_coordinator = (
|
||||
vllm_config.needs_dp_coordinator and not offline_mode and dp_rank == 0
|
||||
)
|
||||
|
||||
if run_coordinator:
|
||||
coordinator = DPCoordinator(parallel_config)
|
||||
coordinator = DPCoordinator(
|
||||
parallel_config,
|
||||
enable_wave_coordination=vllm_config.model_config.is_moe,
|
||||
)
|
||||
|
||||
addresses.coordinator_input, addresses.coordinator_output = (
|
||||
coordinator.get_engine_socket_addresses()
|
||||
@@ -905,6 +923,7 @@ def launch_core_engines(
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
parallel_config,
|
||||
dp_size > 1 and vllm_config.model_config.is_moe,
|
||||
vllm_config.cache_config,
|
||||
local_engine_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
@@ -916,6 +935,7 @@ def wait_for_engine_startup(
|
||||
addresses: EngineZmqAddresses,
|
||||
core_engines: list[CoreEngine],
|
||||
parallel_config: ParallelConfig,
|
||||
coordinated_dp: bool,
|
||||
cache_config: CacheConfig,
|
||||
proc_manager: CoreEngineProcManager | None,
|
||||
coord_process: Process | None,
|
||||
@@ -997,8 +1017,7 @@ def wait_for_engine_startup(
|
||||
)
|
||||
|
||||
if status == "HELLO" and engine.state == CoreEngineState.NEW:
|
||||
# Send init message with DP config info and config hash.
|
||||
# The config hash ensures all DP workers have compatible configs.
|
||||
# Send init message with DP config info.
|
||||
init_message = msgspec.msgpack.encode(
|
||||
EngineHandshakeMetadata(
|
||||
addresses=addresses,
|
||||
@@ -1010,10 +1029,9 @@ def wait_for_engine_startup(
|
||||
"_data_parallel_master_port_list",
|
||||
"data_parallel_size",
|
||||
)
|
||||
},
|
||||
parallel_config_hash=parallel_config.compute_hash()
|
||||
if parallel_config.data_parallel_size > 1
|
||||
else None,
|
||||
}
|
||||
if coordinated_dp
|
||||
else {},
|
||||
)
|
||||
)
|
||||
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
|
||||
@@ -1034,8 +1052,8 @@ def wait_for_engine_startup(
|
||||
if addresses.frontend_stats_publish_address is None:
|
||||
addresses.frontend_stats_publish_address = msg.get("dp_stats_address")
|
||||
|
||||
# Validate config hash consistency across DP workers
|
||||
if parallel_config.data_parallel_size > 1:
|
||||
# Validate config hash consistency across DP workers for MoE models.
|
||||
if coordinated_dp:
|
||||
worker_config_hash = msg.get("parallel_config_hash")
|
||||
expected_hash = parallel_config.compute_hash()
|
||||
if worker_config_hash != expected_hash:
|
||||
|
||||
Reference in New Issue
Block a user