diff --git a/vllm/model_executor/models/granite.py b/vllm/model_executor/models/granite.py index 81ab62937e..24c0851bdf 100644 --- a/vllm/model_executor/models/granite.py +++ b/vllm/model_executor/models/granite.py @@ -60,7 +60,13 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.sequence import IntermediateTensors -from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant +from .interfaces import ( + EagleModelMixin, + SupportsEagle3, + SupportsLoRA, + SupportsPP, + SupportsQuant, +) from .utils import ( AutoWeightsLoader, PPMissingLayer, @@ -307,7 +313,7 @@ class GraniteDecoderLayer(nn.Module): @support_torch_compile -class GraniteModel(nn.Module): +class GraniteModel(nn.Module, EagleModelMixin): hf_to_vllm_mapper = WeightsMapper( orig_to_new_stacked={ # weight_name: (param_name, shard_id) @@ -363,7 +369,7 @@ class GraniteModel(nn.Module): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None, inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: + ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds @@ -375,8 +381,14 @@ class GraniteModel(nn.Module): assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] - for layer in islice(self.layers, self.start_layer, self.end_layer): + aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None) + for idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer) + ): hidden_states = layer(positions, hidden_states) + self._maybe_add_hidden_state( + aux_hidden_states, idx + 1, hidden_states, None + ) if not get_pp_group().is_last_rank: return IntermediateTensors( @@ -386,6 +398,9 @@ class GraniteModel(nn.Module): ) hidden_states = self.norm(hidden_states) + + if len(aux_hidden_states) > 0: + return hidden_states, aux_hidden_states return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -393,7 +408,9 @@ class GraniteModel(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) -class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): +class GraniteForCausalLM( + nn.Module, SupportsLoRA, SupportsPP, SupportsQuant, SupportsEagle3 +): hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper # LoRA specific attributes packed_modules_mapping: dict[str, list[str]] = {