[Model] Support EAGLE3/DFlash aux hidden states for Granite (#60855)

Signed-off-by: Pai, Pramod <pramod.pai@intel.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
devpramod
2026-10-10 13:34:10 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent fa5cfd48d5
commit 46fb84c2d4
+22 -5
View File
@@ -60,7 +60,13 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
from .interfaces import (
EagleModelMixin,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
@@ -307,7 +313,7 @@ class GraniteDecoderLayer(nn.Module):
@support_torch_compile
class GraniteModel(nn.Module):
class GraniteModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
@@ -363,7 +369,7 @@ class GraniteModel(nn.Module):
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -375,8 +381,14 @@ class GraniteModel(nn.Module):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states = layer(positions, hidden_states)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, None
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
@@ -386,6 +398,9 @@ class GraniteModel(nn.Module):
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -393,7 +408,9 @@ class GraniteModel(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
class GraniteForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsQuant, SupportsEagle3
):
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
# LoRA specific attributes
packed_modules_mapping: dict[str, list[str]] = {