[1/N] Elastic EP Milestone 2 (#34861)
Signed-off-by: Yongji Wu <wuyongji317@gmail.com> Signed-off-by: Itay Alroy <ialroy@nvidia.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Ron Tourgeman <rtourgeman@nvidia.com> Co-authored-by: Yongji Wu <wuyongji317@gmail.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
co-authored by
Yongji Wu
Tyler Michael Smith
Ron Tourgeman
parent
90805ff464
commit
dea268336f
@@ -461,6 +461,8 @@ class GPUModelRunner(
|
||||
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
|
||||
|
||||
self.eplb_state: EplbState | None = None
|
||||
# NOTE(yongji): flag to temporarily disable EPLB during scaling up/down
|
||||
self.eep_eplb_suppressed = False
|
||||
"""
|
||||
State of the expert parallelism load balancer.
|
||||
|
||||
@@ -2702,7 +2704,7 @@ class GPUModelRunner(
|
||||
"""
|
||||
Step for the EPLB (Expert Parallelism Load Balancing) state.
|
||||
"""
|
||||
if not self.parallel_config.enable_eplb:
|
||||
if not self.parallel_config.enable_eplb or self.eep_eplb_suppressed:
|
||||
return
|
||||
|
||||
assert self.eplb_state is not None
|
||||
@@ -2714,6 +2716,23 @@ class GPUModelRunner(
|
||||
log_stats=self.parallel_config.eplb_config.log_balancedness,
|
||||
)
|
||||
|
||||
def setup_eplb_from_mapping(
|
||||
self,
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
old_num_physical_experts: int,
|
||||
) -> None:
|
||||
model = self.get_model()
|
||||
assert is_mixture_of_experts(model)
|
||||
|
||||
self.eplb_state = EplbState.from_mapping(
|
||||
model=model,
|
||||
model_config=self.model_config,
|
||||
device=self.device,
|
||||
parallel_config=self.parallel_config,
|
||||
expanded_physical_to_logical=expanded_physical_to_logical,
|
||||
num_valid_physical_experts=old_num_physical_experts,
|
||||
)
|
||||
|
||||
def _pool(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -4175,21 +4194,16 @@ class GPUModelRunner(
|
||||
setattr(self, config_name, new_config)
|
||||
|
||||
@instrument(span_name="Loading (GPU)")
|
||||
def load_model(self, eep_scale_up: bool = False) -> None:
|
||||
def load_model(self, load_dummy_weights: bool = False) -> None:
|
||||
"""
|
||||
Args:
|
||||
eep_scale_up: the model loading is for elastic EP scale up.
|
||||
load_dummy_weights: load dummy weights instead of real weights.
|
||||
"""
|
||||
logger.info_once(
|
||||
"Starting to load model %s...",
|
||||
self.model_config.model,
|
||||
scope="global",
|
||||
)
|
||||
global_expert_loads, old_global_expert_indices_per_model, rank_mapping = (
|
||||
EplbState.get_eep_state(self.parallel_config)
|
||||
if eep_scale_up
|
||||
else (None, None, None)
|
||||
)
|
||||
|
||||
if self.parallel_config.enable_eplb:
|
||||
self.eplb_state = EplbState(self.parallel_config, self.device)
|
||||
@@ -4198,6 +4212,8 @@ class GPUModelRunner(
|
||||
try:
|
||||
with DeviceMemoryProfiler() as m:
|
||||
time_before_load = time.perf_counter()
|
||||
if load_dummy_weights:
|
||||
self.load_config.load_format = "dummy"
|
||||
model_loader = get_model_loader(self.load_config)
|
||||
self.model = model_loader.load_model(
|
||||
vllm_config=self.vllm_config, model_config=self.model_config
|
||||
@@ -4214,6 +4230,9 @@ class GPUModelRunner(
|
||||
and is_mixture_of_experts(self.drafter.model)
|
||||
and self.parallel_config.enable_eplb
|
||||
):
|
||||
assert not self.parallel_config.enable_elastic_ep, (
|
||||
"Elastic EP is not supported with drafter model."
|
||||
)
|
||||
spec_config = self.vllm_config.speculative_config
|
||||
assert spec_config is not None
|
||||
assert spec_config.draft_model_config is not None
|
||||
@@ -4221,17 +4240,6 @@ class GPUModelRunner(
|
||||
"EPLB is enabled for drafter model %s.",
|
||||
spec_config.draft_model_config.model,
|
||||
)
|
||||
|
||||
global_expert_load = (
|
||||
global_expert_loads[eplb_models]
|
||||
if global_expert_loads
|
||||
else None
|
||||
)
|
||||
old_global_expert_indices = (
|
||||
old_global_expert_indices_per_model[eplb_models]
|
||||
if old_global_expert_indices_per_model
|
||||
else None
|
||||
)
|
||||
if self.eplb_state is None:
|
||||
self.eplb_state = EplbState(
|
||||
self.parallel_config, self.device
|
||||
@@ -4239,9 +4247,6 @@ class GPUModelRunner(
|
||||
self.eplb_state.add_model(
|
||||
self.drafter.model,
|
||||
spec_config.draft_model_config,
|
||||
global_expert_load,
|
||||
old_global_expert_indices,
|
||||
rank_mapping,
|
||||
)
|
||||
eplb_models += 1
|
||||
|
||||
@@ -4283,11 +4288,12 @@ class GPUModelRunner(
|
||||
time_after_load - time_before_load,
|
||||
scope="local",
|
||||
)
|
||||
prepare_communication_buffer_for_model(self.model)
|
||||
if (drafter := getattr(self, "drafter", None)) and (
|
||||
drafter_model := getattr(drafter, "model", None)
|
||||
):
|
||||
prepare_communication_buffer_for_model(drafter_model)
|
||||
if not load_dummy_weights:
|
||||
prepare_communication_buffer_for_model(self.model)
|
||||
if (drafter := getattr(self, "drafter", None)) and (
|
||||
drafter_model := getattr(drafter, "model", None)
|
||||
):
|
||||
prepare_communication_buffer_for_model(drafter_model)
|
||||
mm_config = self.model_config.multimodal_config
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
supports_multimodal_pruning(self.get_model())
|
||||
@@ -4295,26 +4301,19 @@ class GPUModelRunner(
|
||||
and mm_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
|
||||
if is_mixture_of_experts(self.model) and self.parallel_config.enable_eplb:
|
||||
if (
|
||||
is_mixture_of_experts(self.model)
|
||||
and self.parallel_config.enable_eplb
|
||||
and not load_dummy_weights
|
||||
):
|
||||
logger.info_once("EPLB is enabled for model %s.", self.model_config.model)
|
||||
global_expert_load = (
|
||||
global_expert_loads[eplb_models] if global_expert_loads else None
|
||||
)
|
||||
old_global_expert_indices = (
|
||||
old_global_expert_indices_per_model[eplb_models]
|
||||
if old_global_expert_indices_per_model
|
||||
else None
|
||||
)
|
||||
assert self.eplb_state is not None
|
||||
self.eplb_state.add_model(
|
||||
self.model,
|
||||
self.model_config,
|
||||
global_expert_load,
|
||||
old_global_expert_indices,
|
||||
rank_mapping,
|
||||
)
|
||||
if self.eplb_state.is_async:
|
||||
self.eplb_state.start_async_loop(rank_mapping=rank_mapping)
|
||||
self.eplb_state.start_async_loop()
|
||||
|
||||
if (
|
||||
self.vllm_config.compilation_config.mode
|
||||
|
||||
Reference in New Issue
Block a user