[1/N] Elastic EP Milestone 2 (#34861)

Signed-off-by: Yongji Wu <wuyongji317@gmail.com>
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Ron Tourgeman <rtourgeman@nvidia.com>
Co-authored-by: Yongji Wu <wuyongji317@gmail.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
Itay Alroy
2026-02-28 04:46:42 +00:00
committed by GitHub
co-authored by Yongji Wu Tyler Michael Smith Ron Tourgeman
parent 90805ff464
commit dea268336f
53 changed files with 3612 additions and 1015 deletions
+39 -40
View File
@@ -461,6 +461,8 @@ class GPUModelRunner(
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
self.eplb_state: EplbState | None = None
# NOTE(yongji): flag to temporarily disable EPLB during scaling up/down
self.eep_eplb_suppressed = False
"""
State of the expert parallelism load balancer.
@@ -2702,7 +2704,7 @@ class GPUModelRunner(
"""
Step for the EPLB (Expert Parallelism Load Balancing) state.
"""
if not self.parallel_config.enable_eplb:
if not self.parallel_config.enable_eplb or self.eep_eplb_suppressed:
return
assert self.eplb_state is not None
@@ -2714,6 +2716,23 @@ class GPUModelRunner(
log_stats=self.parallel_config.eplb_config.log_balancedness,
)
def setup_eplb_from_mapping(
self,
expanded_physical_to_logical: torch.Tensor,
old_num_physical_experts: int,
) -> None:
model = self.get_model()
assert is_mixture_of_experts(model)
self.eplb_state = EplbState.from_mapping(
model=model,
model_config=self.model_config,
device=self.device,
parallel_config=self.parallel_config,
expanded_physical_to_logical=expanded_physical_to_logical,
num_valid_physical_experts=old_num_physical_experts,
)
def _pool(
self,
hidden_states: torch.Tensor,
@@ -4175,21 +4194,16 @@ class GPUModelRunner(
setattr(self, config_name, new_config)
@instrument(span_name="Loading (GPU)")
def load_model(self, eep_scale_up: bool = False) -> None:
def load_model(self, load_dummy_weights: bool = False) -> None:
"""
Args:
eep_scale_up: the model loading is for elastic EP scale up.
load_dummy_weights: load dummy weights instead of real weights.
"""
logger.info_once(
"Starting to load model %s...",
self.model_config.model,
scope="global",
)
global_expert_loads, old_global_expert_indices_per_model, rank_mapping = (
EplbState.get_eep_state(self.parallel_config)
if eep_scale_up
else (None, None, None)
)
if self.parallel_config.enable_eplb:
self.eplb_state = EplbState(self.parallel_config, self.device)
@@ -4198,6 +4212,8 @@ class GPUModelRunner(
try:
with DeviceMemoryProfiler() as m:
time_before_load = time.perf_counter()
if load_dummy_weights:
self.load_config.load_format = "dummy"
model_loader = get_model_loader(self.load_config)
self.model = model_loader.load_model(
vllm_config=self.vllm_config, model_config=self.model_config
@@ -4214,6 +4230,9 @@ class GPUModelRunner(
and is_mixture_of_experts(self.drafter.model)
and self.parallel_config.enable_eplb
):
assert not self.parallel_config.enable_elastic_ep, (
"Elastic EP is not supported with drafter model."
)
spec_config = self.vllm_config.speculative_config
assert spec_config is not None
assert spec_config.draft_model_config is not None
@@ -4221,17 +4240,6 @@ class GPUModelRunner(
"EPLB is enabled for drafter model %s.",
spec_config.draft_model_config.model,
)
global_expert_load = (
global_expert_loads[eplb_models]
if global_expert_loads
else None
)
old_global_expert_indices = (
old_global_expert_indices_per_model[eplb_models]
if old_global_expert_indices_per_model
else None
)
if self.eplb_state is None:
self.eplb_state = EplbState(
self.parallel_config, self.device
@@ -4239,9 +4247,6 @@ class GPUModelRunner(
self.eplb_state.add_model(
self.drafter.model,
spec_config.draft_model_config,
global_expert_load,
old_global_expert_indices,
rank_mapping,
)
eplb_models += 1
@@ -4283,11 +4288,12 @@ class GPUModelRunner(
time_after_load - time_before_load,
scope="local",
)
prepare_communication_buffer_for_model(self.model)
if (drafter := getattr(self, "drafter", None)) and (
drafter_model := getattr(drafter, "model", None)
):
prepare_communication_buffer_for_model(drafter_model)
if not load_dummy_weights:
prepare_communication_buffer_for_model(self.model)
if (drafter := getattr(self, "drafter", None)) and (
drafter_model := getattr(drafter, "model", None)
):
prepare_communication_buffer_for_model(drafter_model)
mm_config = self.model_config.multimodal_config
self.is_multimodal_pruning_enabled = (
supports_multimodal_pruning(self.get_model())
@@ -4295,26 +4301,19 @@ class GPUModelRunner(
and mm_config.is_multimodal_pruning_enabled()
)
if is_mixture_of_experts(self.model) and self.parallel_config.enable_eplb:
if (
is_mixture_of_experts(self.model)
and self.parallel_config.enable_eplb
and not load_dummy_weights
):
logger.info_once("EPLB is enabled for model %s.", self.model_config.model)
global_expert_load = (
global_expert_loads[eplb_models] if global_expert_loads else None
)
old_global_expert_indices = (
old_global_expert_indices_per_model[eplb_models]
if old_global_expert_indices_per_model
else None
)
assert self.eplb_state is not None
self.eplb_state.add_model(
self.model,
self.model_config,
global_expert_load,
old_global_expert_indices,
rank_mapping,
)
if self.eplb_state.is_async:
self.eplb_state.start_async_loop(rank_mapping=rank_mapping)
self.eplb_state.start_async_loop()
if (
self.vllm_config.compilation_config.mode