From 77e1421a68d625bb45ac1d86023be8e555aefeec Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Thu, 14 May 2026 10:26:15 +0800 Subject: [PATCH] [Bugfix] Fix EPLB initialization for VLM wrapper models (#39805) Signed-off-by: esmeetu --- vllm/v1/worker/gpu_model_runner.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 426713e9ca..dc122b05b9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -73,6 +73,7 @@ from vllm.model_executor.model_loader.reload import ( initialize_layerwise_reload, ) from vllm.model_executor.models.interfaces import ( + MixtureOfExperts, MultiModalEmbeddings, SupportsMRoPE, SupportsMultiModal, @@ -489,6 +490,7 @@ class GPUModelRunner( self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode) self.eplb_state: EplbState | None = None + self._moe_model: MixtureOfExperts | None = None # NOTE(yongji): flag to temporarily disable EPLB during scaling up/down self.eep_eplb_suppressed = False """ @@ -3186,8 +3188,7 @@ class GPUModelRunner( return assert self.eplb_state is not None - model = self.get_model() - assert is_mixture_of_experts(model) + assert self._moe_model is not None self.eplb_state.step( is_dummy, is_profile, @@ -3199,11 +3200,10 @@ class GPUModelRunner( expanded_physical_to_logical: torch.Tensor, old_num_physical_experts: int, ) -> None: - model = self.get_model() - assert is_mixture_of_experts(model) + assert self._moe_model is not None self.eplb_state = EplbState.from_mapping( - model=model, + model=self._moe_model, model_config=self.model_config, device=self.device, parallel_config=self.parallel_config, @@ -4996,8 +4996,20 @@ class GPUModelRunner( self.model.set_aux_hidden_state_layers(aux_layers) + # Resolve the MoE model, unwrapping VLM wrappers if needed. + # VLM models (e.g. KimiK25ForConditionalGeneration) wrap the + # actual MoE language model but don't implement + # MixtureOfExperts themselves. + moe_candidate = self.model + if not is_mixture_of_experts(moe_candidate) and isinstance( + moe_candidate, SupportsMultiModal + ): + moe_candidate = moe_candidate.get_language_model() + if is_mixture_of_experts(moe_candidate): + self._moe_model = moe_candidate + if ( - is_mixture_of_experts(self.model) + self._moe_model is not None and self.parallel_config.enable_eplb and not load_dummy_weights ): @@ -5007,7 +5019,7 @@ class GPUModelRunner( ) assert self.eplb_state is not None self.eplb_state.add_model( - self.model, + self._moe_model, self.model_config, ) eplb_models += 1 @@ -5047,7 +5059,7 @@ class GPUModelRunner( ) # Temporary hack for dynamic res video w/o support for bs>1 yet if ( - is_mixture_of_experts(self.model) + self._moe_model is not None and self.parallel_config.enable_eplb and not load_dummy_weights and self.eplb_state is not None