[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+2 -13
View File
@@ -3,11 +3,11 @@
import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
maybe_share_target_embed,
)
@@ -54,18 +54,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
target_inner = getattr(target_language_model, "model", target_language_model)
draft_inner = dflash_model.model
# Skip embedding sharing under PP — each rank owns its own embedding.
if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
maybe_share_target_embed(dflash_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
+4 -16
View File
@@ -4,7 +4,6 @@
import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -43,6 +42,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
maybe_share_target_embed,
)
draft_attention_backend = _resolve_dspark_attention_backend(
@@ -53,6 +53,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
draft_vllm_config = replace(
vllm_config,
parallel_config=speculative_config.draft_parallel_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
@@ -76,9 +77,6 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
vllm_config=draft_vllm_config, model_config=draft_model_config
)
if get_pp_group().world_size != 1:
raise NotImplementedError("DSpark does not support pipeline parallelism.")
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
@@ -88,18 +86,8 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
draft_inner = draft_model.model
target_vocab_size = vllm_config.model_config.get_vocab_size()
target_embed = getattr(target_inner, "embed_tokens", None)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if (
target_embed is not None
and draft_model_config.get_vocab_size() <= target_vocab_size
and _should_share(
draft_model, "has_own_embed_tokens", draft_embed, target_embed
)
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
if draft_model_config.get_vocab_size() <= target_vocab_size:
maybe_share_target_embed(draft_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(draft_model, "lm_head", None)
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import torch
import torch.nn as nn
from vllm.config import SpeculativeConfig
@@ -17,8 +18,6 @@ def set_eagle3_aux_hidden_state_layers(
) -> None:
if not supports_eagle3(model):
raise RuntimeError("Model does not support EAGLE3 interface")
# mypy may infer the class-level overload for supports_eagle3.
# Narrow explicitly to the runtime protocol instance.
if isinstance(model, type):
raise RuntimeError("Expected model instance for EAGLE3 configuration")
eagle3_model = cast(SupportsEagle3, model)
@@ -30,6 +29,72 @@ def set_eagle3_aux_hidden_state_layers(
aux_layers = eagle3_model.get_eagle3_default_aux_hidden_state_layers()
logger.info("Using Eagle3 auxiliary layers from model: %s", aux_layers)
eagle3_model.set_aux_hidden_state_layers(aux_layers)
reserve_aux_intermediate_tensor_slots(model)
def _inner_decoder(model: nn.Module) -> nn.Module | None:
parent_ref = model
if hasattr(model, "get_language_model"):
parent_ref = model.get_language_model()
elif hasattr(model, "language_model"):
parent_ref = model.language_model
return getattr(parent_ref, "model", None)
def verify_supports_aux_hidden_states_over_pp(model: nn.Module, method: str) -> None:
inner = _inner_decoder(model)
if not getattr(inner, "supports_aux_hidden_states_over_pp", False):
raise ValueError(
f"{type(model).__name__} does not support {method} with "
"pipeline parallelism"
)
def aux_hidden_state_relay_keys(model: nn.Module) -> tuple[str, ...]:
from vllm.distributed.parallel_state import get_pp_group
pp = get_pp_group()
if pp.world_size < 2 or pp.is_first_rank or pp.is_last_rank:
return ()
inner = _inner_decoder(model)
assert inner is not None
return tuple(
f"{inner.AUX_HIDDEN_STATE_KEY}{i}" for i in range(inner._aux_slot_base_cached)
)
def reserve_aux_intermediate_tensor_slots(model: nn.Module) -> None:
from vllm.distributed.parallel_state import (
get_pp_group,
model_parallel_is_initialized,
)
if not model_parallel_is_initialized():
return
pp = get_pp_group()
if pp.world_size < 2 or pp.is_first_rank:
return
inner = _inner_decoder(model)
if inner is None or not getattr(inner, "supports_aux_hidden_states_over_pp", False):
return
num_aux_states = inner._aux_slot_base_cached
if num_aux_states == 0:
return
key = inner.AUX_HIDDEN_STATE_KEY
hidden_size = inner.config.hidden_size
make_empty = model.make_empty_intermediate_tensors
def make_empty_with_aux(batch_size, dtype, device):
tensors = make_empty(batch_size, dtype, device)
for i in range(num_aux_states):
tensors[f"{key}{i}"] = torch.zeros(
(batch_size, hidden_size), dtype=dtype, device=device
)
return tensors
model.make_empty_intermediate_tensors = make_empty_with_aux
def get_eagle3_aux_layers_from_config(
+37 -19
View File
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.models.utils import PPMissingLayer
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
@@ -33,6 +34,41 @@ def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module
)
def maybe_share_target_embed(
draft_model: nn.Module, draft_inner: nn.Module, target_inner: nn.Module
) -> None:
"""Share the target input embedding with the drafter when needed."""
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
if isinstance(target_embed, PPMissingLayer):
target_embed = None
# The drafter does not use the target's LoRA adapter.
if isinstance(target_embed, BaseLayerWithLoRA):
target_embed = target_embed.base_layer
draft_embed = getattr(draft_inner, "embed_tokens", None)
if get_pp_group().world_size > 1 and not hasattr(
draft_model, "has_own_embed_tokens"
):
return
if target_embed is None:
if hasattr(draft_inner, "embed_tokens") and not getattr(
draft_model, "has_own_embed_tokens", False
):
raise RuntimeError(
f"{type(draft_model).__name__} needs the target input embedding, "
"but it is unavailable on this PP stage"
)
return
if _should_share(draft_model, "has_own_embed_tokens", draft_embed, target_embed):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag
@@ -70,25 +106,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
target_inner = target_language_model.model
draft_inner = eagle_model.model
# Skip embedding sharing under PP — each rank owns its own embedding.
if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
# If the target's embedding is LoRA-wrapped, share the underlying base
# layer. The draft is not part of the LoRA adapter; sharing the wrapper
# would make the draft run the LoRA embedding kernel with the target's
# punica metadata (sized for the target's token count), causing an
# out-of-bounds GPU access during multi-step draft decode.
if isinstance(target_embed, BaseLayerWithLoRA):
target_embed = target_embed.base_layer
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
maybe_share_target_embed(eagle_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(eagle_model, "lm_head", None)