Elastic Expert Parallel Initial Support (#20775)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
@@ -1745,8 +1745,40 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
new_config = update_config(config, config_overrides)
|
||||
setattr(self, config_name, new_config)
|
||||
|
||||
def load_model(self) -> None:
|
||||
def load_model(self, eep_scale_up: bool = False) -> None:
|
||||
"""
|
||||
Args:
|
||||
eep_scale_up: the model loading is for elastic EP scale up.
|
||||
"""
|
||||
logger.info("Starting to load model %s...", self.model_config.model)
|
||||
if eep_scale_up:
|
||||
from vllm.distributed.parallel_state import get_ep_group
|
||||
num_local_physical_experts = torch.empty(1,
|
||||
dtype=torch.int32,
|
||||
device="cpu")
|
||||
torch.distributed.broadcast(num_local_physical_experts,
|
||||
group=get_ep_group().cpu_group,
|
||||
group_src=0)
|
||||
num_local_physical_experts = int(num_local_physical_experts.item())
|
||||
new_ep_size = get_ep_group().world_size
|
||||
global_expert_load, old_global_expert_indices = (
|
||||
EplbState.recv_state())
|
||||
num_logical_experts = global_expert_load.shape[1]
|
||||
self.parallel_config.num_redundant_experts = (
|
||||
num_local_physical_experts * new_ep_size - num_logical_experts)
|
||||
assert old_global_expert_indices.shape[
|
||||
1] % num_local_physical_experts == 0
|
||||
old_ep_size = old_global_expert_indices.shape[
|
||||
1] // num_local_physical_experts
|
||||
rank_mapping = {
|
||||
old_ep_rank: old_ep_rank
|
||||
for old_ep_rank in range(old_ep_size)
|
||||
}
|
||||
else:
|
||||
global_expert_load = None
|
||||
old_global_expert_indices = None
|
||||
rank_mapping = None
|
||||
|
||||
with DeviceMemoryProfiler() as m: # noqa: SIM117
|
||||
time_before_load = time.perf_counter()
|
||||
model_loader = get_model_loader(self.load_config)
|
||||
@@ -1788,6 +1820,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.model,
|
||||
self.device,
|
||||
self.parallel_config,
|
||||
global_expert_load,
|
||||
old_global_expert_indices,
|
||||
rank_mapping,
|
||||
)
|
||||
|
||||
def save_tensorized_model(
|
||||
|
||||
Reference in New Issue
Block a user