[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+21 -5
View File
@@ -75,6 +75,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -354,6 +355,7 @@ class LlamaModel(nn.Module, EagleModelMixin):
".up_proj": (".gate_up_proj", 1),
}
)
supports_aux_hidden_states_over_pp = True
def __init__(
self,
@@ -372,8 +374,10 @@ class LlamaModel(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
@@ -418,9 +422,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual = layer(
positions, hidden_states, residual, **extra_layer_kwargs
@@ -431,11 +442,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states