[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from tests.utils import multi_gpu_test
from tests.v1.e2e.spec_decode.utils import compute_acceptance_len
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
DRAFT = "nm-testing/Llama3_2_1B_speculator.eagle3"
PROMPTS = [
"The capital of France is",
"2 + 2 equals",
"In one word, the color of the sky is",
"Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:",
]
ACCEPTANCE_TOLERANCE = 0.95
def _run(pp_size: int) -> float:
llm = LLM(
model=MODEL,
tensor_parallel_size=1,
pipeline_parallel_size=pp_size,
max_model_len=512,
gpu_memory_utilization=0.45,
disable_log_stats=False,
compilation_config={"cudagraph_mode": "FULL_AND_PIECEWISE"},
speculative_config={
"method": "eagle3",
"model": DRAFT,
"num_speculative_tokens": 3,
},
)
try:
llm.generate(
PROMPTS,
SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True),
)
acceptance = compute_acceptance_len(llm.get_metrics())
assert acceptance > 1
return acceptance
finally:
del llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@multi_gpu_test(num_gpus=4)
def test_eagle3_pipeline_parallel_acceptance():
baseline = _run(1)
for pp_size in (2, 4):
parallel = _run(pp_size)
assert parallel >= baseline * ACCEPTANCE_TOLERANCE, (
f"PP={pp_size} acceptance regressed: {parallel:.3f} < "
f"{baseline:.3f} * {ACCEPTANCE_TOLERANCE}"
)
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm.model_executor.models.interfaces import EagleModelMixin
from vllm.model_executor.models.mimo import MiMoModel
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
verify_supports_aux_hidden_states_over_pp,
)
def test_aux_layers_are_sorted_and_deduplicated():
model = EagleModelMixin()
model._set_aux_hidden_state_layers((48, 3, 90, 24, 48))
assert model.aux_hidden_state_layers == (3, 24, 48, 90)
def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
inner = MiMoModel.__new__(MiMoModel)
target = SimpleNamespace(model=inner)
assert not inner.supports_aux_hidden_states_over_pp
with pytest.raises(ValueError, match="does not support eagle3"):
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.utils import (
PPMissingLayer,
spec_decode_needs_target_embed,
)
from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils
VOCAB, HIDDEN = 32, 8
def _fake_pp(world_size: int, is_last_rank: bool = True):
return lambda: SimpleNamespace(
world_size=world_size,
is_last_rank=is_last_rank,
is_first_rank=world_size == 1,
)
def _inner(embed: nn.Module | None) -> nn.Module:
inner = nn.Module()
if embed is not None:
inner.embed_tokens = embed
return inner
def _embed(fill: float | None = None) -> nn.Embedding:
embed = nn.Embedding(VOCAB, HIDDEN)
if fill is not None:
with torch.no_grad():
embed.weight.fill_(fill)
return embed
@pytest.mark.parametrize("draft_embed", ["loaded", "unset"])
def test_drafter_without_own_embedding_gets_the_targets(monkeypatch, draft_embed):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
target_embed = _embed()
draft_inner = _inner(_embed() if draft_embed == "loaded" else None)
if draft_embed == "unset":
draft_inner.embed_tokens = None
draft = SimpleNamespace(has_own_embed_tokens=False)
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(target_embed))
assert draft_inner.embed_tokens is target_embed
def test_missing_target_embedding_raises_instead_of_running_on_garbage(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_inner = _inner(_embed())
draft = SimpleNamespace(has_own_embed_tokens=False)
with pytest.raises(RuntimeError, match="needs the target input embedding"):
eagle_utils.maybe_share_target_embed(
draft, draft_inner, _inner(PPMissingLayer())
)
def test_drafter_with_distinct_weights_keeps_them(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_embed = _embed(fill=1.0)
draft_inner = _inner(draft_embed)
draft = SimpleNamespace(has_own_embed_tokens=True)
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(_embed(fill=2.0)))
assert draft_inner.embed_tokens is draft_embed
def test_mtp_style_drafter_is_left_alone_under_pp(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_embed = _embed()
draft_inner = _inner(draft_embed)
eagle_utils.maybe_share_target_embed(nn.Module(), draft_inner, _inner(_embed()))
assert draft_inner.embed_tokens is draft_embed
@pytest.mark.parametrize(
"method,pp_size,is_last_rank,expected",
[
("eagle", 2, True, True),
("eagle3", 2, True, True),
("dflash", 2, True, True),
("dspark", 2, True, True),
("eagle3", 1, True, False),
("eagle3", 2, False, False),
("mtp", 2, True, False),
(None, 2, True, False),
],
)
def test_target_embedding_provisioning(
monkeypatch, method, pp_size, is_last_rank, expected
):
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
_fake_pp(pp_size, is_last_rank),
raising=True,
)
speculative_config = None if method is None else SimpleNamespace(method=method)
config = SimpleNamespace(speculative_config=speculative_config)
assert spec_decode_needs_target_embed(config) is expected