[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -30,12 +30,10 @@ WorkerType = Literal["existing", "new", "removing"]
|
||||
|
||||
|
||||
class ScaleUpExistingEngineState(enum.IntEnum):
|
||||
CREATE_STANDBY_GROUPS = 0
|
||||
STAGE_QUANT_METHODS = 1
|
||||
TRANSFER_WEIGHTS = 2
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 3
|
||||
COMMIT_SCALE_UP = 4 # Blocks forward passes.
|
||||
COMPLETE = 5
|
||||
PREPARE = 0
|
||||
SYNC_KV_CACHE_MEMORY_SIZE = 1
|
||||
COMMIT_SCALE_UP = 2 # Blocks forward passes.
|
||||
COMPLETE = 3
|
||||
|
||||
|
||||
class ScaleUpNewEngineState(enum.IntEnum):
|
||||
@@ -95,7 +93,7 @@ class ElasticEPScalingState:
|
||||
self.state = (
|
||||
ScaleUpNewEngineState.PRE_KV_INIT
|
||||
if worker_type == "new"
|
||||
else ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS
|
||||
else ScaleUpExistingEngineState.PREPARE
|
||||
)
|
||||
else:
|
||||
self.state = (
|
||||
@@ -166,20 +164,8 @@ class ElasticEPScalingState:
|
||||
state = self.state
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleUpExistingEngineState.CREATE_STANDBY_GROUPS:
|
||||
if not self._create_standby_groups():
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.STAGE_QUANT_METHODS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.STAGE_QUANT_METHODS:
|
||||
if not self._execute_async("stage_standby_moe_quant_methods"):
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.TRANSFER_WEIGHTS
|
||||
return True
|
||||
|
||||
elif state == ScaleUpExistingEngineState.TRANSFER_WEIGHTS:
|
||||
if not self._transfer_weights():
|
||||
if state == ScaleUpExistingEngineState.PREPARE:
|
||||
if not self._prepare_workers():
|
||||
return False
|
||||
self.state = ScaleUpExistingEngineState.SYNC_KV_CACHE_MEMORY_SIZE
|
||||
return True
|
||||
@@ -210,7 +196,7 @@ class ElasticEPScalingState:
|
||||
assert self.new_dp_group is not None and self.new_dp_store is not None
|
||||
|
||||
if state == ScaleUpNewEngineState.PRE_KV_INIT:
|
||||
self._collective_rpc("elastic_ep_execute", args=("receive_weights",))
|
||||
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
|
||||
self.engine_core.available_gpu_memory_for_kv_cache = (
|
||||
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
|
||||
)
|
||||
@@ -243,7 +229,7 @@ class ElasticEPScalingState:
|
||||
assert self.old_dp_group is not None
|
||||
|
||||
if state == ScaleDownRemainingEngineState.PREPARE:
|
||||
if self._create_standby_groups():
|
||||
if self._prepare_workers():
|
||||
self.state = ScaleDownRemainingEngineState.COMMIT_SCALE_DOWN
|
||||
self._mark_ready_for_switch()
|
||||
return True
|
||||
@@ -328,29 +314,18 @@ class ElasticEPScalingState:
|
||||
self._prepare_future = None
|
||||
return True
|
||||
|
||||
def _create_standby_groups(self) -> bool:
|
||||
def _prepare_workers(self) -> bool:
|
||||
assert self.old_dp_group is not None
|
||||
if not self._ensure_new_dp_group():
|
||||
return False
|
||||
if not self._execute_async(
|
||||
"create_standby_groups",
|
||||
"prepare_reconfiguration",
|
||||
self.reconfig_request,
|
||||
self.new_parallel_config.use_all2all,
|
||||
):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Created standby communication groups")
|
||||
return True
|
||||
|
||||
def _transfer_weights(self) -> bool:
|
||||
assert self.reconfig_request is not None and self.old_dp_group is not None
|
||||
old_dp_size = self.old_dp_group.size()
|
||||
new_dp_size = self.reconfig_request.new_data_parallel_size
|
||||
|
||||
if not self._execute_async("transfer_weights", old_dp_size, new_dp_size):
|
||||
return False
|
||||
if self.old_dp_group.rank() == 0:
|
||||
logger.info("[Elastic EP] Transferred weights to new workers")
|
||||
logger.info("[Elastic EP] Prepared reconfiguration")
|
||||
return True
|
||||
|
||||
def _sync_kv_cache_memory_size(self) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user