Elastic Expert Parallel Initial Support (#20775)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
Rui Qiao
2025-07-18 17:46:09 -07:00
committed by GitHub
parent 5782581acf
commit 217937221b
24 changed files with 1659 additions and 68 deletions
+36 -1
View File
@@ -1745,8 +1745,40 @@ class GPUModelRunner(LoRAModelRunnerMixin):
new_config = update_config(config, config_overrides)
setattr(self, config_name, new_config)
def load_model(self) -> None:
def load_model(self, eep_scale_up: bool = False) -> None:
"""
Args:
eep_scale_up: the model loading is for elastic EP scale up.
"""
logger.info("Starting to load model %s...", self.model_config.model)
if eep_scale_up:
from vllm.distributed.parallel_state import get_ep_group
num_local_physical_experts = torch.empty(1,
dtype=torch.int32,
device="cpu")
torch.distributed.broadcast(num_local_physical_experts,
group=get_ep_group().cpu_group,
group_src=0)
num_local_physical_experts = int(num_local_physical_experts.item())
new_ep_size = get_ep_group().world_size
global_expert_load, old_global_expert_indices = (
EplbState.recv_state())
num_logical_experts = global_expert_load.shape[1]
self.parallel_config.num_redundant_experts = (
num_local_physical_experts * new_ep_size - num_logical_experts)
assert old_global_expert_indices.shape[
1] % num_local_physical_experts == 0
old_ep_size = old_global_expert_indices.shape[
1] // num_local_physical_experts
rank_mapping = {
old_ep_rank: old_ep_rank
for old_ep_rank in range(old_ep_size)
}
else:
global_expert_load = None
old_global_expert_indices = None
rank_mapping = None
with DeviceMemoryProfiler() as m: # noqa: SIM117
time_before_load = time.perf_counter()
model_loader = get_model_loader(self.load_config)
@@ -1788,6 +1820,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.model,
self.device,
self.parallel_config,
global_expert_load,
old_global_expert_indices,
rank_mapping,
)
def save_tensorized_model(