[Model] Support EAGLE3/DFlash aux hidden states for Granite (#60855)
Signed-off-by: Pai, Pramod <pramod.pai@intel.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fa5cfd48d5
commit
46fb84c2d4
@@ -60,7 +60,13 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -307,7 +313,7 @@ class GraniteDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class GraniteModel(nn.Module):
|
||||
class GraniteModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
@@ -363,7 +369,7 @@ class GraniteModel(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -375,8 +381,14 @@ class GraniteModel(nn.Module):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, None
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
@@ -386,6 +398,9 @@ class GraniteModel(nn.Module):
|
||||
)
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -393,7 +408,9 @@ class GraniteModel(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
class GraniteForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsQuant, SupportsEagle3
|
||||
):
|
||||
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
|
||||
# LoRA specific attributes
|
||||
packed_modules_mapping: dict[str, list[str]] = {
|
||||
|
||||
Reference in New Issue
Block a user