From 0a508743d42a26786c1432bb7f2e93f8111b6383 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Wed, 20 May 2026 09:15:52 -0400 Subject: [PATCH] [Spec Decode] Support non-MTP speculation for NemotronH (#43130) Signed-off-by: Benjamin Chislett --- vllm/model_executor/models/nemotron_h.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index 537e19afbc..8915d8172c 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -64,9 +64,12 @@ from vllm.model_executor.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from vllm.model_executor.models.interfaces import ( + EagleModelMixin, HasInnerState, IsHybrid, MixtureOfExperts, + SupportsEagle, + SupportsEagle3, SupportsLoRA, SupportsMambaPrefixCaching, SupportsPP, @@ -539,7 +542,7 @@ ALL_DECODER_LAYER_TYPES = { @support_torch_compile -class NemotronHModel(nn.Module): +class NemotronHModel(nn.Module, EagleModelMixin): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -605,18 +608,27 @@ class NemotronHModel(nn.Module): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - for layer in islice(self.layers, self.start_layer, self.end_layer): + aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) + for idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer) + ): hidden_states, residual = layer( positions=positions, hidden_states=hidden_states, residual=residual, ) + self._maybe_add_hidden_state( + aux_hidden_states, idx + 1, hidden_states, residual + ) if not get_pp_group().is_last_rank: return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) hidden_states, _ = self.norm_f(hidden_states, residual) + + if len(aux_hidden_states) > 0: + return hidden_states, aux_hidden_states return hidden_states def is_spec_layer(self, config: NemotronHConfig, weight_name: str) -> bool: @@ -767,6 +779,8 @@ class NemotronHForCausalLM( HasInnerState, SupportsLoRA, SupportsPP, + SupportsEagle, + SupportsEagle3, IsHybrid, SupportsQuant, MixtureOfExperts,