From 12421962955ac28b6f80a0307f554fad939174dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fa=C3=A7=20Eldenk?= Date: Tue, 19 May 2026 15:39:00 -0500 Subject: [PATCH] [Model] Support post-norm architecture for EAGLE-3 supeculators (#42764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Doğaç Eldenk --- vllm/model_executor/models/deepseek_eagle3.py | 36 ++++++++++-- vllm/model_executor/models/llama_eagle3.py | 57 +++++++++++++++---- vllm/v1/worker/gpu_model_runner.py | 5 ++ 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/vllm/model_executor/models/deepseek_eagle3.py b/vllm/model_executor/models/deepseek_eagle3.py index 47dfc06706..4d26a9e009 100644 --- a/vllm/model_executor/models/deepseek_eagle3.py +++ b/vllm/model_executor/models/deepseek_eagle3.py @@ -199,11 +199,18 @@ class DeepseekV2Eagle3Model(nn.Module): ] ) - # fc layer for combining auxiliary hidden states (3x hidden size input) - if hasattr(self.config, "target_hidden_size"): - fc_input_size = self.config.target_hidden_size * 3 - else: - fc_input_size = self.config.hidden_size * 3 + # fc layer for combining auxiliary hidden states + num_aux_hidden_states = getattr(self.config, "num_aux_hidden_states", None) + if num_aux_hidden_states is None: + eagle_config = getattr(self.config, "eagle_config", None) or {} + layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids") + num_aux_hidden_states = len(layer_ids) if layer_ids else 3 + self.num_aux_hidden_states = num_aux_hidden_states + + target_hidden_size = getattr( + self.config, "target_hidden_size", self.config.hidden_size + ) + fc_input_size = target_hidden_size * num_aux_hidden_states self.fc = ReplicatedLinear( input_size=fc_input_size, @@ -215,6 +222,18 @@ class DeepseekV2Eagle3Model(nn.Module): return_bias=False, ) + use_fc_norm = getattr(self.config, "fc_norm", False) + if use_fc_norm: + self.fc_norm = nn.ModuleList( + [ + RMSNorm(target_hidden_size, eps=self.config.rms_norm_eps) + for _ in range(self.num_aux_hidden_states) + ] + ) + else: + self.fc_norm = None + + self.norm_output = getattr(self.config, "norm_output", False) self.norm = RMSNorm( self.config.hidden_size, eps=self.config.rms_norm_eps, @@ -242,8 +261,13 @@ class DeepseekV2Eagle3Model(nn.Module): hidden_states=hidden_states, residual=residual, ) + hidden_states, hidden_prenorm = self.norm(hidden_states, residual) - return hidden_states, hidden_prenorm + + # norm_output variant uses the post-norm hidden states. + aux_output = hidden_states if self.norm_output else hidden_prenorm + + return hidden_states, aux_output def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index af115c4567..ed75b4c7c2 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -172,19 +172,40 @@ class LlamaModel(nn.Module): ] ) if self.use_aux_hidden_state: - if hasattr(self.config, "target_hidden_size"): - fc_input_size = self.config.target_hidden_size * 3 - else: - fc_input_size = self.config.hidden_size * 3 + self.num_aux_hidden_states = getattr( + self.config, "num_aux_hidden_states", None + ) + if self.num_aux_hidden_states is None: + eagle_config = getattr(self.config, "eagle_config", None) or {} + layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids") + self.num_aux_hidden_states = len(layer_ids) if layer_ids else 3 + + target_hidden_size = getattr( + self.config, "target_hidden_size", self.config.hidden_size + ) + self.fc_input_size = target_hidden_size * self.num_aux_hidden_states + if self.norm_before_fc: self.input_norm = RMSNorm( - fc_input_size, + self.fc_input_size, eps=self.config.rms_norm_eps, ) else: self.input_norm = None + + use_fc_norm = getattr(self.config, "fc_norm", False) + if use_fc_norm: + self.fc_norm = nn.ModuleList( + [ + RMSNorm(target_hidden_size, eps=self.config.rms_norm_eps) + for _ in range(self.num_aux_hidden_states) + ] + ) + else: + self.fc_norm = None + self.fc = ReplicatedLinear( - input_size=fc_input_size, + input_size=self.fc_input_size, output_size=self.config.hidden_size, bias=False, params_dtype=vllm_config.model_config.dtype, @@ -192,6 +213,8 @@ class LlamaModel(nn.Module): prefix=maybe_prefix(prefix, "fc"), return_bias=False, ) + + self.norm_output = getattr(self.config, "norm_output", False) self.norm = RMSNorm( self.config.hidden_size, eps=self.config.rms_norm_eps, @@ -220,7 +243,11 @@ class LlamaModel(nn.Module): residual=residual, ) hidden_states, hidden_prenorm = self.norm(hidden_states, residual) - return hidden_states, hidden_prenorm + + # norm_output variant uses the post-norm hidden states. + aux_output = hidden_states if self.norm_output else hidden_prenorm + + return hidden_states, aux_output def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ @@ -312,11 +339,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM): if self.use_parallel_drafting: self.register_buffer( "mask_hidden", - torch.zeros( - 1, - (3 if self.model.use_aux_hidden_state else 1) - * self.config.hidden_size, - ), + torch.zeros(1, self.model.fc_input_size), persistent=False, ) @@ -371,6 +394,16 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM): if self.model.norm_before_fc: hidden_states = self.model.input_norm(hidden_states) + + # `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm` + # applies separate RMSNorms to each chunk of the hidden states. + if self.model.fc_norm is not None: + chunks = hidden_states.chunk(self.model.num_aux_hidden_states, dim=-1) + hidden_states = torch.cat( + [norm(chunk) for norm, chunk in zip(self.model.fc_norm, chunks)], + dim=-1, + ) + return self.model.fc(hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index c4871d90be..61e705ceb7 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5210,9 +5210,14 @@ class GPUModelRunner( layer_ids = getattr(hf_config, "eagle_aux_hidden_state_layer_ids", None) if not layer_ids: dflash_config = getattr(hf_config, "dflash_config", None) + eagle_config = getattr(hf_config, "eagle_config", None) + if dflash_config and isinstance(dflash_config, dict): layer_ids = dflash_config.get("target_layer_ids") + if eagle_config and isinstance(eagle_config, dict): + layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids") + if layer_ids and isinstance(layer_ids, (list, tuple)): return tuple(layer_ids)