[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -3,11 +3,11 @@
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
maybe_share_target_embed,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,18 +54,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
target_inner = getattr(target_language_model, "model", target_language_model)
|
||||
draft_inner = dflash_model.model
|
||||
|
||||
# Skip embedding sharing under PP — each rank owns its own embedding.
|
||||
if get_pp_group().world_size == 1:
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
maybe_share_target_embed(dflash_model, draft_inner, target_inner)
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -43,6 +42,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
maybe_share_target_embed,
|
||||
)
|
||||
|
||||
draft_attention_backend = _resolve_dspark_attention_backend(
|
||||
@@ -53,6 +53,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
parallel_config=speculative_config.draft_parallel_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
@@ -76,9 +77,6 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
vllm_config=draft_vllm_config, model_config=draft_model_config
|
||||
)
|
||||
|
||||
if get_pp_group().world_size != 1:
|
||||
raise NotImplementedError("DSpark does not support pipeline parallelism.")
|
||||
|
||||
target_language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
@@ -88,18 +86,8 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
draft_inner = draft_model.model
|
||||
target_vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
|
||||
target_embed = getattr(target_inner, "embed_tokens", None)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if (
|
||||
target_embed is not None
|
||||
and draft_model_config.get_vocab_size() <= target_vocab_size
|
||||
and _should_share(
|
||||
draft_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
)
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
if draft_model_config.get_vocab_size() <= target_vocab_size:
|
||||
maybe_share_target_embed(draft_model, draft_inner, target_inner)
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(draft_model, "lm_head", None)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import SpeculativeConfig
|
||||
@@ -17,8 +18,6 @@ def set_eagle3_aux_hidden_state_layers(
|
||||
) -> None:
|
||||
if not supports_eagle3(model):
|
||||
raise RuntimeError("Model does not support EAGLE3 interface")
|
||||
# mypy may infer the class-level overload for supports_eagle3.
|
||||
# Narrow explicitly to the runtime protocol instance.
|
||||
if isinstance(model, type):
|
||||
raise RuntimeError("Expected model instance for EAGLE3 configuration")
|
||||
eagle3_model = cast(SupportsEagle3, model)
|
||||
@@ -30,6 +29,72 @@ def set_eagle3_aux_hidden_state_layers(
|
||||
aux_layers = eagle3_model.get_eagle3_default_aux_hidden_state_layers()
|
||||
logger.info("Using Eagle3 auxiliary layers from model: %s", aux_layers)
|
||||
eagle3_model.set_aux_hidden_state_layers(aux_layers)
|
||||
reserve_aux_intermediate_tensor_slots(model)
|
||||
|
||||
|
||||
def _inner_decoder(model: nn.Module) -> nn.Module | None:
|
||||
parent_ref = model
|
||||
if hasattr(model, "get_language_model"):
|
||||
parent_ref = model.get_language_model()
|
||||
elif hasattr(model, "language_model"):
|
||||
parent_ref = model.language_model
|
||||
return getattr(parent_ref, "model", None)
|
||||
|
||||
|
||||
def verify_supports_aux_hidden_states_over_pp(model: nn.Module, method: str) -> None:
|
||||
inner = _inner_decoder(model)
|
||||
if not getattr(inner, "supports_aux_hidden_states_over_pp", False):
|
||||
raise ValueError(
|
||||
f"{type(model).__name__} does not support {method} with "
|
||||
"pipeline parallelism"
|
||||
)
|
||||
|
||||
|
||||
def aux_hidden_state_relay_keys(model: nn.Module) -> tuple[str, ...]:
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
|
||||
pp = get_pp_group()
|
||||
if pp.world_size < 2 or pp.is_first_rank or pp.is_last_rank:
|
||||
return ()
|
||||
inner = _inner_decoder(model)
|
||||
assert inner is not None
|
||||
return tuple(
|
||||
f"{inner.AUX_HIDDEN_STATE_KEY}{i}" for i in range(inner._aux_slot_base_cached)
|
||||
)
|
||||
|
||||
|
||||
def reserve_aux_intermediate_tensor_slots(model: nn.Module) -> None:
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
|
||||
if not model_parallel_is_initialized():
|
||||
return
|
||||
pp = get_pp_group()
|
||||
if pp.world_size < 2 or pp.is_first_rank:
|
||||
return
|
||||
inner = _inner_decoder(model)
|
||||
if inner is None or not getattr(inner, "supports_aux_hidden_states_over_pp", False):
|
||||
return
|
||||
|
||||
num_aux_states = inner._aux_slot_base_cached
|
||||
if num_aux_states == 0:
|
||||
return
|
||||
|
||||
key = inner.AUX_HIDDEN_STATE_KEY
|
||||
hidden_size = inner.config.hidden_size
|
||||
make_empty = model.make_empty_intermediate_tensors
|
||||
|
||||
def make_empty_with_aux(batch_size, dtype, device):
|
||||
tensors = make_empty(batch_size, dtype, device)
|
||||
for i in range(num_aux_states):
|
||||
tensors[f"{key}{i}"] = torch.zeros(
|
||||
(batch_size, hidden_size), dtype=dtype, device=device
|
||||
)
|
||||
return tensors
|
||||
|
||||
model.make_empty_intermediate_tensors = make_empty_with_aux
|
||||
|
||||
|
||||
def get_eagle3_aux_layers_from_config(
|
||||
|
||||
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
|
||||
|
||||
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
@@ -33,6 +34,41 @@ def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module
|
||||
)
|
||||
|
||||
|
||||
def maybe_share_target_embed(
|
||||
draft_model: nn.Module, draft_inner: nn.Module, target_inner: nn.Module
|
||||
) -> None:
|
||||
"""Share the target input embedding with the drafter when needed."""
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
if isinstance(target_embed, PPMissingLayer):
|
||||
target_embed = None
|
||||
# The drafter does not use the target's LoRA adapter.
|
||||
if isinstance(target_embed, BaseLayerWithLoRA):
|
||||
target_embed = target_embed.base_layer
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
|
||||
if get_pp_group().world_size > 1 and not hasattr(
|
||||
draft_model, "has_own_embed_tokens"
|
||||
):
|
||||
return
|
||||
|
||||
if target_embed is None:
|
||||
if hasattr(draft_inner, "embed_tokens") and not getattr(
|
||||
draft_model, "has_own_embed_tokens", False
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{type(draft_model).__name__} needs the target input embedding, "
|
||||
"but it is unavailable on this PP stage"
|
||||
)
|
||||
return
|
||||
|
||||
if _should_share(draft_model, "has_own_embed_tokens", draft_embed, target_embed):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
|
||||
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
@@ -70,25 +106,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
target_inner = target_language_model.model
|
||||
draft_inner = eagle_model.model
|
||||
|
||||
# Skip embedding sharing under PP — each rank owns its own embedding.
|
||||
if get_pp_group().world_size == 1:
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
# If the target's embedding is LoRA-wrapped, share the underlying base
|
||||
# layer. The draft is not part of the LoRA adapter; sharing the wrapper
|
||||
# would make the draft run the LoRA embedding kernel with the target's
|
||||
# punica metadata (sized for the target's token count), causing an
|
||||
# out-of-bounds GPU access during multi-step draft decode.
|
||||
if isinstance(target_embed, BaseLayerWithLoRA):
|
||||
target_embed = target_embed.base_layer
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
maybe_share_target_embed(eagle_model, draft_inner, target_inner)
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(eagle_model, "lm_head", None)
|
||||
|
||||
Reference in New Issue
Block a user