[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -347,6 +347,7 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft_model_config,
|
||||
draft_parallel_config=SimpleNamespace(),
|
||||
attention_backend=None,
|
||||
kv_cache_dtype=None,
|
||||
),
|
||||
@@ -362,9 +363,8 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
|
||||
with (
|
||||
patch.object(dspark_utils, "replace", side_effect=fake_replace),
|
||||
patch.object(
|
||||
dspark_utils,
|
||||
"get_pp_group",
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.eagle.utils.get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
patch(
|
||||
|
||||
@@ -157,25 +157,23 @@ def test_last_layer_on_the_final_rank_uses_the_output_aggregation(
|
||||
assert recorder[0].kwargs["num_blocks"] == 99
|
||||
|
||||
|
||||
def test_last_layer_of_a_non_final_stage_falls_back(recorder, monkeypatch):
|
||||
"""The consumer lives on the next rank and the output aggregation only
|
||||
exists on the last one, so there is nothing here to mix against.
|
||||
|
||||
This is the case that would otherwise reach for weights this rank never
|
||||
constructs. The forward guard is `layer_idx + 1 < end_layer`, where
|
||||
`end_layer` is the rank's own exclusive bound from `get_pp_indices`, so a
|
||||
`PPMissingLayer` is unreachable by construction -- the fallback below is
|
||||
what makes that true rather than merely likely.
|
||||
"""
|
||||
def test_aux_layer_at_non_final_pp_boundary_is_rejected(monkeypatch):
|
||||
model = k3_model.KimiLinearModel.__new__(k3_model.KimiLinearModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_attn_res = True
|
||||
model.end_layer = 72
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.model_parallel_is_initialized", lambda: False
|
||||
)
|
||||
_set_last_rank(monkeypatch, False)
|
||||
prefix_sum = torch.tensor([3.0, 4.0])
|
||||
|
||||
got = _call(
|
||||
_stub_model(enabled=True), END_LAYER - 1, prefix_sum, None, torch.zeros(2)
|
||||
monkeypatch.setattr(
|
||||
k3_model.KimiLinearModel,
|
||||
"_aux_attn_res_stream",
|
||||
property(lambda self: True),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, prefix_sum)
|
||||
assert not recorder, "no weights exist on this rank to mix against"
|
||||
with pytest.raises(ValueError, match="Auxiliary layer 72"):
|
||||
model._set_aux_hidden_state_layers((3, 24, 48, 72, 90))
|
||||
|
||||
|
||||
def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from tests.v1.e2e.spec_decode.utils import compute_acceptance_len
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
DRAFT = "nm-testing/Llama3_2_1B_speculator.eagle3"
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
"2 + 2 equals",
|
||||
"In one word, the color of the sky is",
|
||||
"Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:",
|
||||
]
|
||||
|
||||
ACCEPTANCE_TOLERANCE = 0.95
|
||||
|
||||
|
||||
def _run(pp_size: int) -> float:
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=pp_size,
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.45,
|
||||
disable_log_stats=False,
|
||||
compilation_config={"cudagraph_mode": "FULL_AND_PIECEWISE"},
|
||||
speculative_config={
|
||||
"method": "eagle3",
|
||||
"model": DRAFT,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
)
|
||||
try:
|
||||
llm.generate(
|
||||
PROMPTS,
|
||||
SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True),
|
||||
)
|
||||
acceptance = compute_acceptance_len(llm.get_metrics())
|
||||
assert acceptance > 1
|
||||
return acceptance
|
||||
finally:
|
||||
del llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_eagle3_pipeline_parallel_acceptance():
|
||||
baseline = _run(1)
|
||||
for pp_size in (2, 4):
|
||||
parallel = _run(pp_size)
|
||||
assert parallel >= baseline * ACCEPTANCE_TOLERANCE, (
|
||||
f"PP={pp_size} acceptance regressed: {parallel:.3f} < "
|
||||
f"{baseline:.3f} * {ACCEPTANCE_TOLERANCE}"
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.interfaces import EagleModelMixin
|
||||
from vllm.model_executor.models.mimo import MiMoModel
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
verify_supports_aux_hidden_states_over_pp,
|
||||
)
|
||||
|
||||
|
||||
def test_aux_layers_are_sorted_and_deduplicated():
|
||||
model = EagleModelMixin()
|
||||
model._set_aux_hidden_state_layers((48, 3, 90, 24, 48))
|
||||
assert model.aux_hidden_state_layers == (3, 24, 48, 90)
|
||||
|
||||
|
||||
def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
|
||||
inner = MiMoModel.__new__(MiMoModel)
|
||||
target = SimpleNamespace(model=inner)
|
||||
|
||||
assert not inner.supports_aux_hidden_states_over_pp
|
||||
with pytest.raises(ValueError, match="does not support eagle3"):
|
||||
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils
|
||||
|
||||
VOCAB, HIDDEN = 32, 8
|
||||
|
||||
|
||||
def _fake_pp(world_size: int, is_last_rank: bool = True):
|
||||
return lambda: SimpleNamespace(
|
||||
world_size=world_size,
|
||||
is_last_rank=is_last_rank,
|
||||
is_first_rank=world_size == 1,
|
||||
)
|
||||
|
||||
|
||||
def _inner(embed: nn.Module | None) -> nn.Module:
|
||||
inner = nn.Module()
|
||||
if embed is not None:
|
||||
inner.embed_tokens = embed
|
||||
return inner
|
||||
|
||||
|
||||
def _embed(fill: float | None = None) -> nn.Embedding:
|
||||
embed = nn.Embedding(VOCAB, HIDDEN)
|
||||
if fill is not None:
|
||||
with torch.no_grad():
|
||||
embed.weight.fill_(fill)
|
||||
return embed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_embed", ["loaded", "unset"])
|
||||
def test_drafter_without_own_embedding_gets_the_targets(monkeypatch, draft_embed):
|
||||
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
|
||||
target_embed = _embed()
|
||||
draft_inner = _inner(_embed() if draft_embed == "loaded" else None)
|
||||
if draft_embed == "unset":
|
||||
draft_inner.embed_tokens = None
|
||||
draft = SimpleNamespace(has_own_embed_tokens=False)
|
||||
|
||||
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(target_embed))
|
||||
|
||||
assert draft_inner.embed_tokens is target_embed
|
||||
|
||||
|
||||
def test_missing_target_embedding_raises_instead_of_running_on_garbage(monkeypatch):
|
||||
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
|
||||
draft_inner = _inner(_embed())
|
||||
draft = SimpleNamespace(has_own_embed_tokens=False)
|
||||
|
||||
with pytest.raises(RuntimeError, match="needs the target input embedding"):
|
||||
eagle_utils.maybe_share_target_embed(
|
||||
draft, draft_inner, _inner(PPMissingLayer())
|
||||
)
|
||||
|
||||
|
||||
def test_drafter_with_distinct_weights_keeps_them(monkeypatch):
|
||||
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
|
||||
draft_embed = _embed(fill=1.0)
|
||||
draft_inner = _inner(draft_embed)
|
||||
draft = SimpleNamespace(has_own_embed_tokens=True)
|
||||
|
||||
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(_embed(fill=2.0)))
|
||||
|
||||
assert draft_inner.embed_tokens is draft_embed
|
||||
|
||||
|
||||
def test_mtp_style_drafter_is_left_alone_under_pp(monkeypatch):
|
||||
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
|
||||
draft_embed = _embed()
|
||||
draft_inner = _inner(draft_embed)
|
||||
|
||||
eagle_utils.maybe_share_target_embed(nn.Module(), draft_inner, _inner(_embed()))
|
||||
|
||||
assert draft_inner.embed_tokens is draft_embed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,pp_size,is_last_rank,expected",
|
||||
[
|
||||
("eagle", 2, True, True),
|
||||
("eagle3", 2, True, True),
|
||||
("dflash", 2, True, True),
|
||||
("dspark", 2, True, True),
|
||||
("eagle3", 1, True, False),
|
||||
("eagle3", 2, False, False),
|
||||
("mtp", 2, True, False),
|
||||
(None, 2, True, False),
|
||||
],
|
||||
)
|
||||
def test_target_embedding_provisioning(
|
||||
monkeypatch, method, pp_size, is_last_rank, expected
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_pp_group",
|
||||
_fake_pp(pp_size, is_last_rank),
|
||||
raising=True,
|
||||
)
|
||||
speculative_config = None if method is None else SimpleNamespace(method=method)
|
||||
config = SimpleNamespace(speculative_config=speculative_config)
|
||||
assert spec_decode_needs_target_embed(config) is expected
|
||||
Reference in New Issue
Block a user