[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -3,11 +3,11 @@
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
maybe_share_target_embed,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,18 +54,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
target_inner = getattr(target_language_model, "model", target_language_model)
|
||||
draft_inner = dflash_model.model
|
||||
|
||||
# Skip embedding sharing under PP — each rank owns its own embedding.
|
||||
if get_pp_group().world_size == 1:
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
):
|
||||
if draft_embed is not None:
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
maybe_share_target_embed(dflash_model, draft_inner, target_inner)
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
|
||||
Reference in New Issue
Block a user