[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-08-19 02:13:06 +00:00
committed by GitHub
parent d36cc4254e
commit a2257f95b7
12 changed files with 194 additions and 303 deletions
+12 -37
View File
@@ -30,12 +30,10 @@ WorkerType = Literal["existing", "new", "removing"]
class ScaleUpExistingEngineState(enum.IntEnum):
CREATE_STANDBY_GROUPS = 0
STAGE_QUANT_METHODS = 1
TRANSFER_WEIGHTS = 2
SYNC_KV_CACHE_MEMORY_SIZE = 3
COMMIT_SCALE_UP = 4 # Blocks forward passes.
COMPLETE = 5
PREPARE = 0
SYNC_KV_CACHE_MEMORY_SIZE = 1
COMMIT_SCALE_UP = 2 # Blocks forward passes.
COMPLETE = 3
class ScaleUpNewEngineState(enum.IntEnum):
@@ -95,7 +93,7 @@ class ElasticEPScalingState:
self.state = (
ScaleUpNewEngineState.PRE_KV_INIT
if worker_type == "new"
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
else ScaleUpExistingEngineState.PREPARE
)
else:
self.state = (
@@ -166,20 +164,8 @@ class ElasticEPScalingState:
state = self.state
assert self.old_dp_group is not None
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
if not self._create_standby_groups():
return False
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
return True
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
if not self._execute_async("stage_standby_moe_quant_methods"):
return False
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
return True
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
if not self._transfer_weights():
if state == ScaleUpExistingEngineState.PREPARE:
if not self._prepare_workers():
return False
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
return True
@@ -210,7 +196,7 @@ class ElasticEPScalingState:
assert self.new_dp_group is not None and self.new_dp_store is not None
if state == ScaleUpNewEngineState.PRE_KV_INIT:
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
self.engine_core.available_gpu_memory_for_kv_cache = (
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
)
@@ -243,7 +229,7 @@ class ElasticEPScalingState:
assert self.old_dp_group is not None
if state == ScaleDownRemainingEngineState.PREPARE:
if self._create_standby_groups():
if self._prepare_workers():
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
self._mark_ready_for_switch()
return True
@@ -328,29 +314,18 @@ class ElasticEPScalingState:
self._prepare_future = None
return True
def _create_standby_groups(self) -> bool:
def _prepare_workers(self) -> bool:
assert self.old_dp_group is not None
if not self._ensure_new_dp_group():
return False
if not self._execute_async(
"create_standby_groups",
"prepare_reconfiguration",
self.reconfig_request,
self.new_parallel_config.use_all2all,
):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Created standby communication groups")
return True
def _transfer_weights(self) -> bool:
assert self.reconfig_request is not None and self.old_dp_group is not None
old_dp_size = self.old_dp_group.size()
new_dp_size = self.reconfig_request.new_data_parallel_size
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
return False
if self.old_dp_group.rank() == 0:
logger.info("[Elastic EP] Transferred weights to new workers")
logger.info("[Elastic EP] Prepared reconfiguration")
return True
def _sync_kv_cache_memory_size(self) -> bool: