[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
@@ -285,9 +285,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config
self.config.target_layer_count = target_layer_num
+58 -1
View File
@@ -3,6 +3,7 @@
import asyncio
import weakref
from bisect import bisect_right
from collections.abc import (
AsyncGenerator,
Callable,
@@ -1572,10 +1573,36 @@ class LocalArgmaxMixin:
class EagleModelMixin:
start_layer: int
aux_hidden_state_layers: tuple[int, ...] = ()
supports_aux_hidden_states_over_pp: ClassVar[bool] = False
AUX_HIDDEN_STATE_KEY: ClassVar[str] = "aux_hidden_states_"
_aux_slot_base_cached: int = 0
_aux_upstream_total_cached: int = 0
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.aux_hidden_state_layers = layers
self.aux_hidden_state_layers = tuple(sorted(set(layers)))
self._aux_slot_base_cached = 0
self._aux_upstream_total_cached = 0
self._cache_aux_pp_layout()
def _cache_aux_pp_layout(self) -> None:
from vllm.distributed.parallel_state import (
get_pp_group,
model_parallel_is_initialized,
)
if not model_parallel_is_initialized():
return
pp = get_pp_group()
if pp.world_size < 2:
return
if not pp.is_first_rank:
self._aux_slot_base_cached = bisect_right(
self.aux_hidden_state_layers, self.start_layer
)
if pp.is_last_rank:
self._aux_upstream_total_cached = self._aux_slot_base_cached
def _maybe_add_hidden_state(
self,
@@ -1589,6 +1616,36 @@ class EagleModelMixin:
aux_hidden_states.append(value)
return aux_hidden_states
def pack_local_aux_hidden_states(
self, aux_hidden_states: list[torch.Tensor]
) -> dict[str, torch.Tensor]:
if not aux_hidden_states:
return {}
base = self._aux_slot_base_cached
return {
f"{self.AUX_HIDDEN_STATE_KEY}{base + i}": t
for i, t in enumerate(aux_hidden_states)
}
def collect_remote_aux_hidden_states(
self, intermediate_tensors: "IntermediateTensors | None"
) -> list[torch.Tensor]:
total = self._aux_upstream_total_cached
if total == 0:
return []
assert intermediate_tensors is not None
out: list[torch.Tensor] = []
for i in range(total):
key = f"{self.AUX_HIDDEN_STATE_KEY}{i}"
if key not in intermediate_tensors.tensors:
raise RuntimeError(
f"Missing {key} from PP intermediate tensors: "
f"{sorted(intermediate_tensors.tensors)}"
)
out.append(intermediate_tensors[key])
return out
@runtime_checkable
class SupportsEagle(SupportsEagleBase, Protocol):
+1 -3
View File
@@ -245,9 +245,7 @@ class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
self.has_own_embed_tokens = False
self.has_own_lm_head = False
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.config.target_layer_count = target_layer_num
target_vocab_size = vllm_config.model_config.get_vocab_size()
if self.config.draft_vocab_size != target_vocab_size:
+21 -5
View File
@@ -75,6 +75,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -354,6 +355,7 @@ class LlamaModel(nn.Module, EagleModelMixin):
".up_proj": (".gate_up_proj", 1),
}
)
supports_aux_hidden_states_over_pp = True
def __init__(
self,
@@ -372,8 +374,10 @@ class LlamaModel(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
@@ -418,9 +422,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual = layer(
positions, hidden_states, residual, **extra_layer_kwargs
@@ -431,11 +442,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+1 -3
View File
@@ -314,9 +314,7 @@ class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
if getattr(self.config, "draft_vocab_size", None) is None:
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config for
# proper layer_types indexing in draft models
+2
View File
@@ -55,6 +55,8 @@ logger = init_logger(__name__)
}
)
class MiMoModel(Qwen2Model):
supports_aux_hidden_states_over_pp = False
def forward(
self,
input_ids: torch.Tensor | None,
+21 -5
View File
@@ -74,6 +74,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -330,6 +331,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
".up_proj": (".gate_up_proj", 1),
}
)
supports_aux_hidden_states_over_pp = True
def __init__(
self,
@@ -360,8 +362,10 @@ class Qwen2Model(nn.Module, EagleModelMixin):
self.quant_config = quant_config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -412,9 +416,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
@@ -423,11 +434,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
+1 -3
View File
@@ -694,9 +694,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = self.model_cls(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
+1 -3
View File
@@ -182,9 +182,7 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = Qwen3DSparkModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
+1 -3
View File
@@ -288,9 +288,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
if getattr(self.config, "draft_vocab_size", None) is None:
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config for
# proper layer_types indexing in draft models
+16
View File
@@ -835,6 +835,22 @@ class PPMissingLayer(torch.nn.Identity):
return args[0] if args else next(iter(kwargs.values()))
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
"""Whether the last PP rank needs the target input embedding."""
from vllm.distributed.parallel_state import get_pp_group
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.method not in (
"eagle",
"eagle3",
"dflash",
"dspark",
):
return False
pp = get_pp_group()
return pp.world_size > 1 and pp.is_last_rank
def make_layers(
num_hidden_layers: int,
layer_fn: LayerFn,