[Refactor] Consolidate SupportsEagle (#36063)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
@@ -32,7 +32,13 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -170,7 +176,7 @@ class ArceeDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class ArceeModel(nn.Module):
|
||||
class ArceeModel(nn.Module, EagleModelMixin):
|
||||
"""The transformer model backbone for Arcee (embedding layer + stacked
|
||||
decoder blocks + final norm)."""
|
||||
|
||||
@@ -218,10 +224,6 @@ class ArceeModel(nn.Module):
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
# For optional capturing of intermediate hidden states
|
||||
# (not used by default)
|
||||
self.aux_hidden_state_layers: tuple[int, ...] = tuple()
|
||||
|
||||
# Prepare factory for empty intermediate tensors
|
||||
# (for pipeline scheduling)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
@@ -253,15 +255,14 @@ class ArceeModel(nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(
|
||||
hidden_states + residual
|
||||
) # capture pre-layer hidden state if needed
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, residual
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
# Send intermediate results to the next pipeline stage
|
||||
@@ -348,7 +349,9 @@ class ArceeModel(nn.Module):
|
||||
return loaded_params
|
||||
|
||||
|
||||
class ArceeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
class ArceeForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
"""Arcee Model for causal language modeling, integrated with vLLM
|
||||
runtime."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user