[Model] Support post-norm architecture for EAGLE-3 supeculators (#42764)

Signed-off-by: Doğaç Eldenk <dogacel@gmail.com>
This commit is contained in:
Doğaç Eldenk
2026-05-19 13:39:00 -07:00
committed by GitHub
parent a65093c1a3
commit 1242196295
3 changed files with 80 additions and 18 deletions
+30 -6
View File
@@ -199,11 +199,18 @@ class DeepseekV2Eagle3Model(nn.Module):
]
)
# fc layer for combining auxiliary hidden states (3x hidden size input)
if hasattr(self.config, "target_hidden_size"):
fc_input_size = self.config.target_hidden_size * 3
else:
fc_input_size = self.config.hidden_size * 3
# fc layer for combining auxiliary hidden states
num_aux_hidden_states = getattr(self.config, "num_aux_hidden_states", None)
if num_aux_hidden_states is None:
eagle_config = getattr(self.config, "eagle_config", None) or {}
layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids")
num_aux_hidden_states = len(layer_ids) if layer_ids else 3
self.num_aux_hidden_states = num_aux_hidden_states
target_hidden_size = getattr(
self.config, "target_hidden_size", self.config.hidden_size
)
fc_input_size = target_hidden_size * num_aux_hidden_states
self.fc = ReplicatedLinear(
input_size=fc_input_size,
@@ -215,6 +222,18 @@ class DeepseekV2Eagle3Model(nn.Module):
return_bias=False,
)
use_fc_norm = getattr(self.config, "fc_norm", False)
if use_fc_norm:
self.fc_norm = nn.ModuleList(
[
RMSNorm(target_hidden_size, eps=self.config.rms_norm_eps)
for _ in range(self.num_aux_hidden_states)
]
)
else:
self.fc_norm = None
self.norm_output = getattr(self.config, "norm_output", False)
self.norm = RMSNorm(
self.config.hidden_size,
eps=self.config.rms_norm_eps,
@@ -242,8 +261,13 @@ class DeepseekV2Eagle3Model(nn.Module):
hidden_states=hidden_states,
residual=residual,
)
hidden_states, hidden_prenorm = self.norm(hidden_states, residual)
return hidden_states, hidden_prenorm
# norm_output variant uses the post-norm hidden states.
aux_output = hidden_states if self.norm_output else hidden_prenorm
return hidden_states, aux_output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
+45 -12
View File
@@ -172,19 +172,40 @@ class LlamaModel(nn.Module):
]
)
if self.use_aux_hidden_state:
if hasattr(self.config, "target_hidden_size"):
fc_input_size = self.config.target_hidden_size * 3
else:
fc_input_size = self.config.hidden_size * 3
self.num_aux_hidden_states = getattr(
self.config, "num_aux_hidden_states", None
)
if self.num_aux_hidden_states is None:
eagle_config = getattr(self.config, "eagle_config", None) or {}
layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids")
self.num_aux_hidden_states = len(layer_ids) if layer_ids else 3
target_hidden_size = getattr(
self.config, "target_hidden_size", self.config.hidden_size
)
self.fc_input_size = target_hidden_size * self.num_aux_hidden_states
if self.norm_before_fc:
self.input_norm = RMSNorm(
fc_input_size,
self.fc_input_size,
eps=self.config.rms_norm_eps,
)
else:
self.input_norm = None
use_fc_norm = getattr(self.config, "fc_norm", False)
if use_fc_norm:
self.fc_norm = nn.ModuleList(
[
RMSNorm(target_hidden_size, eps=self.config.rms_norm_eps)
for _ in range(self.num_aux_hidden_states)
]
)
else:
self.fc_norm = None
self.fc = ReplicatedLinear(
input_size=fc_input_size,
input_size=self.fc_input_size,
output_size=self.config.hidden_size,
bias=False,
params_dtype=vllm_config.model_config.dtype,
@@ -192,6 +213,8 @@ class LlamaModel(nn.Module):
prefix=maybe_prefix(prefix, "fc"),
return_bias=False,
)
self.norm_output = getattr(self.config, "norm_output", False)
self.norm = RMSNorm(
self.config.hidden_size,
eps=self.config.rms_norm_eps,
@@ -220,7 +243,11 @@ class LlamaModel(nn.Module):
residual=residual,
)
hidden_states, hidden_prenorm = self.norm(hidden_states, residual)
return hidden_states, hidden_prenorm
# norm_output variant uses the post-norm hidden states.
aux_output = hidden_states if self.norm_output else hidden_prenorm
return hidden_states, aux_output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
@@ -312,11 +339,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
if self.use_parallel_drafting:
self.register_buffer(
"mask_hidden",
torch.zeros(
1,
(3 if self.model.use_aux_hidden_state else 1)
* self.config.hidden_size,
),
torch.zeros(1, self.model.fc_input_size),
persistent=False,
)
@@ -371,6 +394,16 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
if self.model.norm_before_fc:
hidden_states = self.model.input_norm(hidden_states)
# `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm`
# applies separate RMSNorms to each chunk of the hidden states.
if self.model.fc_norm is not None:
chunks = hidden_states.chunk(self.model.num_aux_hidden_states, dim=-1)
hidden_states = torch.cat(
[norm(chunk) for norm, chunk in zip(self.model.fc_norm, chunks)],
dim=-1,
)
return self.model.fc(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+5
View File
@@ -5210,9 +5210,14 @@ class GPUModelRunner(
layer_ids = getattr(hf_config, "eagle_aux_hidden_state_layer_ids", None)
if not layer_ids:
dflash_config = getattr(hf_config, "dflash_config", None)
eagle_config = getattr(hf_config, "eagle_config", None)
if dflash_config and isinstance(dflash_config, dict):
layer_ids = dflash_config.get("target_layer_ids")
if eagle_config and isinstance(eagle_config, dict):
layer_ids = eagle_config.get("eagle_aux_hidden_state_layer_ids")
if layer_ids and isinstance(layer_ids, (list, tuple)):
return tuple(layer_ids)