[Model] Support speculative decoding method for PLaMo3 (#54239)

Signed-off-by: Shinichi Hemmi <shemmi@preferred.jp>
This commit is contained in:
Shinichi Hemmi
2026-08-28 22:31:07 +00:00
committed by GitHub
parent fd98d32f73
commit df14152ac6
2 changed files with 101 additions and 10 deletions
+29 -10
View File
@@ -35,7 +35,12 @@ from vllm.model_executor.model_loader.weight_utils import (
composed_weight_loader,
default_weight_loader,
)
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
extract_layer_index,
@@ -285,7 +290,7 @@ class Plamo3DecoderLayer(nn.Module):
return hidden_states, residual
class Plamo3Decoder(torch.nn.Module):
class Plamo3Decoder(torch.nn.Module, EagleModelMixin):
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
num_hidden_layers = vllm_config.model_config.hf_config.num_hidden_layers
@@ -296,23 +301,32 @@ class Plamo3Decoder(torch.nn.Module):
prefix=f"{prefix}.layers",
)
def __len__(self) -> int:
return len(self.layers)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
for layer in islice(self.layers, self.start_layer, self.end_layer):
) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]:
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
return hidden_states, residual
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, residual
)
return hidden_states, residual, aux_hidden_states
@support_torch_compile
class Plamo3Model(nn.Module):
class Plamo3Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -340,13 +354,16 @@ class Plamo3Model(nn.Module):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.layers._set_aux_hidden_state_layers(layers)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -358,7 +375,7 @@ class Plamo3Model(nn.Module):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
hidden_states, residual = self.layers(
hidden_states, residual, aux_hidden_states = self.layers(
positions=positions, hidden_states=hidden_states, residual=residual
)
if not get_pp_group().is_last_rank:
@@ -366,10 +383,12 @@ class Plamo3Model(nn.Module):
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
packed_modules_mapping = {
"qkv_proj": ["qkv_proj"],
"gate_up_proj": ["gate_up_proj"],
@@ -416,7 +435,7 @@ class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)