[Bugfix] Fix EPLB initialization for VLM wrapper models (#39805)
Signed-off-by: esmeetu <jasonailu87@gmail.com>
This commit is contained in:
@@ -73,6 +73,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -489,6 +490,7 @@ class GPUModelRunner(
|
||||
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
|
||||
|
||||
self.eplb_state: EplbState | None = None
|
||||
self._moe_model: MixtureOfExperts | None = None
|
||||
# NOTE(yongji): flag to temporarily disable EPLB during scaling up/down
|
||||
self.eep_eplb_suppressed = False
|
||||
"""
|
||||
@@ -3186,8 +3188,7 @@ class GPUModelRunner(
|
||||
return
|
||||
|
||||
assert self.eplb_state is not None
|
||||
model = self.get_model()
|
||||
assert is_mixture_of_experts(model)
|
||||
assert self._moe_model is not None
|
||||
self.eplb_state.step(
|
||||
is_dummy,
|
||||
is_profile,
|
||||
@@ -3199,11 +3200,10 @@ class GPUModelRunner(
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
old_num_physical_experts: int,
|
||||
) -> None:
|
||||
model = self.get_model()
|
||||
assert is_mixture_of_experts(model)
|
||||
assert self._moe_model is not None
|
||||
|
||||
self.eplb_state = EplbState.from_mapping(
|
||||
model=model,
|
||||
model=self._moe_model,
|
||||
model_config=self.model_config,
|
||||
device=self.device,
|
||||
parallel_config=self.parallel_config,
|
||||
@@ -4996,8 +4996,20 @@ class GPUModelRunner(
|
||||
|
||||
self.model.set_aux_hidden_state_layers(aux_layers)
|
||||
|
||||
# Resolve the MoE model, unwrapping VLM wrappers if needed.
|
||||
# VLM models (e.g. KimiK25ForConditionalGeneration) wrap the
|
||||
# actual MoE language model but don't implement
|
||||
# MixtureOfExperts themselves.
|
||||
moe_candidate = self.model
|
||||
if not is_mixture_of_experts(moe_candidate) and isinstance(
|
||||
moe_candidate, SupportsMultiModal
|
||||
):
|
||||
moe_candidate = moe_candidate.get_language_model()
|
||||
if is_mixture_of_experts(moe_candidate):
|
||||
self._moe_model = moe_candidate
|
||||
|
||||
if (
|
||||
is_mixture_of_experts(self.model)
|
||||
self._moe_model is not None
|
||||
and self.parallel_config.enable_eplb
|
||||
and not load_dummy_weights
|
||||
):
|
||||
@@ -5007,7 +5019,7 @@ class GPUModelRunner(
|
||||
)
|
||||
assert self.eplb_state is not None
|
||||
self.eplb_state.add_model(
|
||||
self.model,
|
||||
self._moe_model,
|
||||
self.model_config,
|
||||
)
|
||||
eplb_models += 1
|
||||
@@ -5047,7 +5059,7 @@ class GPUModelRunner(
|
||||
) # Temporary hack for dynamic res video w/o support for bs>1 yet
|
||||
|
||||
if (
|
||||
is_mixture_of_experts(self.model)
|
||||
self._moe_model is not None
|
||||
and self.parallel_config.enable_eplb
|
||||
and not load_dummy_weights
|
||||
and self.eplb_state is not None
|
||||
|
||||
Reference in New Issue
Block a user