[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -835,6 +835,22 @@ class PPMissingLayer(torch.nn.Identity):
|
||||
return args[0] if args else next(iter(kwargs.values()))
|
||||
|
||||
|
||||
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the last PP rank needs the target input embedding."""
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.method not in (
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
):
|
||||
return False
|
||||
pp = get_pp_group()
|
||||
return pp.world_size > 1 and pp.is_last_rank
|
||||
|
||||
|
||||
def make_layers(
|
||||
num_hidden_layers: int,
|
||||
layer_fn: LayerFn,
|
||||
|
||||
Reference in New Issue
Block a user