[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+3 -3
View File
@@ -347,6 +347,7 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=draft_model_config,
draft_parallel_config=SimpleNamespace(),
attention_backend=None,
kv_cache_dtype=None,
),
@@ -362,9 +363,8 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
with (
patch.object(dspark_utils, "replace", side_effect=fake_replace),
patch.object(
dspark_utils,
"get_pp_group",
patch(
"vllm.v1.worker.gpu.spec_decode.eagle.utils.get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
patch(
@@ -157,25 +157,23 @@ def test_last_layer_on_the_final_rank_uses_the_output_aggregation(
assert recorder[0].kwargs["num_blocks"] == 99
def test_last_layer_of_a_non_final_stage_falls_back(recorder, monkeypatch):
"""The consumer lives on the next rank and the output aggregation only
exists on the last one, so there is nothing here to mix against.
This is the case that would otherwise reach for weights this rank never
constructs. The forward guard is `layer_idx + 1 < end_layer`, where
`end_layer` is the rank's own exclusive bound from `get_pp_indices`, so a
`PPMissingLayer` is unreachable by construction -- the fallback below is
what makes that true rather than merely likely.
"""
def test_aux_layer_at_non_final_pp_boundary_is_rejected(monkeypatch):
model = k3_model.KimiLinearModel.__new__(k3_model.KimiLinearModel)
torch.nn.Module.__init__(model)
model.use_attn_res = True
model.end_layer = 72
monkeypatch.setattr(
"vllm.distributed.parallel_state.model_parallel_is_initialized", lambda: False
)
_set_last_rank(monkeypatch, False)
prefix_sum = torch.tensor([3.0, 4.0])
got = _call(
_stub_model(enabled=True), END_LAYER - 1, prefix_sum, None, torch.zeros(2)
monkeypatch.setattr(
k3_model.KimiLinearModel,
"_aux_attn_res_stream",
property(lambda self: True),
)
torch.testing.assert_close(got, prefix_sum)
assert not recorder, "no weights exist on this rank to mix against"
with pytest.raises(ValueError, match="Auxiliary layer 72"):
model._set_aux_hidden_state_layers((3, 24, 48, 72, 90))
def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from tests.utils import multi_gpu_test
from tests.v1.e2e.spec_decode.utils import compute_acceptance_len
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
DRAFT = "nm-testing/Llama3_2_1B_speculator.eagle3"
PROMPTS = [
"The capital of France is",
"2 + 2 equals",
"In one word, the color of the sky is",
"Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:",
]
ACCEPTANCE_TOLERANCE = 0.95
def _run(pp_size: int) -> float:
llm = LLM(
model=MODEL,
tensor_parallel_size=1,
pipeline_parallel_size=pp_size,
max_model_len=512,
gpu_memory_utilization=0.45,
disable_log_stats=False,
compilation_config={"cudagraph_mode": "FULL_AND_PIECEWISE"},
speculative_config={
"method": "eagle3",
"model": DRAFT,
"num_speculative_tokens": 3,
},
)
try:
llm.generate(
PROMPTS,
SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True),
)
acceptance = compute_acceptance_len(llm.get_metrics())
assert acceptance > 1
return acceptance
finally:
del llm
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@multi_gpu_test(num_gpus=4)
def test_eagle3_pipeline_parallel_acceptance():
baseline = _run(1)
for pp_size in (2, 4):
parallel = _run(pp_size)
assert parallel >= baseline * ACCEPTANCE_TOLERANCE, (
f"PP={pp_size} acceptance regressed: {parallel:.3f} < "
f"{baseline:.3f} * {ACCEPTANCE_TOLERANCE}"
)
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm.model_executor.models.interfaces import EagleModelMixin
from vllm.model_executor.models.mimo import MiMoModel
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
verify_supports_aux_hidden_states_over_pp,
)
def test_aux_layers_are_sorted_and_deduplicated():
model = EagleModelMixin()
model._set_aux_hidden_state_layers((48, 3, 90, 24, 48))
assert model.aux_hidden_state_layers == (3, 24, 48, 90)
def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
inner = MiMoModel.__new__(MiMoModel)
target = SimpleNamespace(model=inner)
assert not inner.supports_aux_hidden_states_over_pp
with pytest.raises(ValueError, match="does not support eagle3"):
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.utils import (
PPMissingLayer,
spec_decode_needs_target_embed,
)
from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils
VOCAB, HIDDEN = 32, 8
def _fake_pp(world_size: int, is_last_rank: bool = True):
return lambda: SimpleNamespace(
world_size=world_size,
is_last_rank=is_last_rank,
is_first_rank=world_size == 1,
)
def _inner(embed: nn.Module | None) -> nn.Module:
inner = nn.Module()
if embed is not None:
inner.embed_tokens = embed
return inner
def _embed(fill: float | None = None) -> nn.Embedding:
embed = nn.Embedding(VOCAB, HIDDEN)
if fill is not None:
with torch.no_grad():
embed.weight.fill_(fill)
return embed
@pytest.mark.parametrize("draft_embed", ["loaded", "unset"])
def test_drafter_without_own_embedding_gets_the_targets(monkeypatch, draft_embed):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
target_embed = _embed()
draft_inner = _inner(_embed() if draft_embed == "loaded" else None)
if draft_embed == "unset":
draft_inner.embed_tokens = None
draft = SimpleNamespace(has_own_embed_tokens=False)
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(target_embed))
assert draft_inner.embed_tokens is target_embed
def test_missing_target_embedding_raises_instead_of_running_on_garbage(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_inner = _inner(_embed())
draft = SimpleNamespace(has_own_embed_tokens=False)
with pytest.raises(RuntimeError, match="needs the target input embedding"):
eagle_utils.maybe_share_target_embed(
draft, draft_inner, _inner(PPMissingLayer())
)
def test_drafter_with_distinct_weights_keeps_them(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_embed = _embed(fill=1.0)
draft_inner = _inner(draft_embed)
draft = SimpleNamespace(has_own_embed_tokens=True)
eagle_utils.maybe_share_target_embed(draft, draft_inner, _inner(_embed(fill=2.0)))
assert draft_inner.embed_tokens is draft_embed
def test_mtp_style_drafter_is_left_alone_under_pp(monkeypatch):
monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2))
draft_embed = _embed()
draft_inner = _inner(draft_embed)
eagle_utils.maybe_share_target_embed(nn.Module(), draft_inner, _inner(_embed()))
assert draft_inner.embed_tokens is draft_embed
@pytest.mark.parametrize(
"method,pp_size,is_last_rank,expected",
[
("eagle", 2, True, True),
("eagle3", 2, True, True),
("dflash", 2, True, True),
("dspark", 2, True, True),
("eagle3", 1, True, False),
("eagle3", 2, False, False),
("mtp", 2, True, False),
(None, 2, True, False),
],
)
def test_target_embedding_provisioning(
monkeypatch, method, pp_size, is_last_rank, expected
):
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
_fake_pp(pp_size, is_last_rank),
raising=True,
)
speculative_config = None if method is None else SimpleNamespace(method=method)
config = SimpleNamespace(speculative_config=speculative_config)
assert spec_decode_needs_target_embed(config) is expected
+1 -1
View File
@@ -1699,7 +1699,7 @@ class SpeculativeConfig:
This is mostly a copy of the target parallel config, except the tp_size.
"""
draft_parallel_config = ParallelConfig(
pipeline_parallel_size=target_parallel_config.pipeline_parallel_size,
pipeline_parallel_size=1,
tensor_parallel_size=speculative_draft_tensor_parallel_size,
distributed_executor_backend=target_parallel_config.distributed_executor_backend,
max_parallel_loading_workers=target_parallel_config.max_parallel_loading_workers,
-6
View File
@@ -2590,12 +2590,6 @@ class VllmConfig:
):
unsupported.append("parallel drafting for EAGLE speculative decoding")
if (
speculative_config.method == "eagle3"
and self.parallel_config.pipeline_parallel_size > 1
):
unsupported.append("EAGLE3 with pipeline parallelism")
if self.parallel_config.use_ubatching:
unsupported.extend(self._get_dbo_unsupported_features())
@@ -285,9 +285,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config
self.config.target_layer_count = target_layer_num
+58 -1
View File
@@ -3,6 +3,7 @@
import asyncio
import weakref
from bisect import bisect_right
from collections.abc import (
AsyncGenerator,
Callable,
@@ -1572,10 +1573,36 @@ class LocalArgmaxMixin:
class EagleModelMixin:
start_layer: int
aux_hidden_state_layers: tuple[int, ...] = ()
supports_aux_hidden_states_over_pp: ClassVar[bool] = False
AUX_HIDDEN_STATE_KEY: ClassVar[str] = "aux_hidden_states_"
_aux_slot_base_cached: int = 0
_aux_upstream_total_cached: int = 0
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.aux_hidden_state_layers = layers
self.aux_hidden_state_layers = tuple(sorted(set(layers)))
self._aux_slot_base_cached = 0
self._aux_upstream_total_cached = 0
self._cache_aux_pp_layout()
def _cache_aux_pp_layout(self) -> None:
from vllm.distributed.parallel_state import (
get_pp_group,
model_parallel_is_initialized,
)
if not model_parallel_is_initialized():
return
pp = get_pp_group()
if pp.world_size < 2:
return
if not pp.is_first_rank:
self._aux_slot_base_cached = bisect_right(
self.aux_hidden_state_layers, self.start_layer
)
if pp.is_last_rank:
self._aux_upstream_total_cached = self._aux_slot_base_cached
def _maybe_add_hidden_state(
self,
@@ -1589,6 +1616,36 @@ class EagleModelMixin:
aux_hidden_states.append(value)
return aux_hidden_states
def pack_local_aux_hidden_states(
self, aux_hidden_states: list[torch.Tensor]
) -> dict[str, torch.Tensor]:
if not aux_hidden_states:
return {}
base = self._aux_slot_base_cached
return {
f"{self.AUX_HIDDEN_STATE_KEY}{base + i}": t
for i, t in enumerate(aux_hidden_states)
}
def collect_remote_aux_hidden_states(
self, intermediate_tensors: "IntermediateTensors | None"
) -> list[torch.Tensor]:
total = self._aux_upstream_total_cached
if total == 0:
return []
assert intermediate_tensors is not None
out: list[torch.Tensor] = []
for i in range(total):
key = f"{self.AUX_HIDDEN_STATE_KEY}{i}"
if key not in intermediate_tensors.tensors:
raise RuntimeError(
f"Missing {key} from PP intermediate tensors: "
f"{sorted(intermediate_tensors.tensors)}"
)
out.append(intermediate_tensors[key])
return out
@runtime_checkable
class SupportsEagle(SupportsEagleBase, Protocol):
+1 -3
View File
@@ -245,9 +245,7 @@ class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
self.has_own_embed_tokens = False
self.has_own_lm_head = False
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.config.target_layer_count = target_layer_num
target_vocab_size = vllm_config.model_config.get_vocab_size()
if self.config.draft_vocab_size != target_vocab_size:
+21 -5
View File
@@ -75,6 +75,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -354,6 +355,7 @@ class LlamaModel(nn.Module, EagleModelMixin):
".up_proj": (".gate_up_proj", 1),
}
)
supports_aux_hidden_states_over_pp = True
def __init__(
self,
@@ -372,8 +374,10 @@ class LlamaModel(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
@@ -418,9 +422,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual = layer(
positions, hidden_states, residual, **extra_layer_kwargs
@@ -431,11 +442,16 @@ class LlamaModel(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+1 -3
View File
@@ -314,9 +314,7 @@ class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
if getattr(self.config, "draft_vocab_size", None) is None:
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config for
# proper layer_types indexing in draft models
+2
View File
@@ -55,6 +55,8 @@ logger = init_logger(__name__)
}
)
class MiMoModel(Qwen2Model):
supports_aux_hidden_states_over_pp = False
def forward(
self,
input_ids: torch.Tensor | None,
+21 -5
View File
@@ -74,6 +74,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -330,6 +331,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
".up_proj": (".gate_up_proj", 1),
}
)
supports_aux_hidden_states_over_pp = True
def __init__(
self,
@@ -360,8 +362,10 @@ class Qwen2Model(nn.Module, EagleModelMixin):
self.quant_config = quant_config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (config.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
@@ -412,9 +416,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
@@ -423,11 +434,16 @@ class Qwen2Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
+1 -3
View File
@@ -694,9 +694,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = self.model_cls(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
+1 -3
View File
@@ -182,9 +182,7 @@ class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = Qwen3DSparkModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
+1 -3
View File
@@ -288,9 +288,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
if getattr(self.config, "draft_vocab_size", None) is None:
base_vocab_size = getattr(self.config, "vocab_size", None)
self.config.draft_vocab_size = base_vocab_size
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
# Store target layer count in draft config for
# proper layer_types indexing in draft models
+16
View File
@@ -835,6 +835,22 @@ class PPMissingLayer(torch.nn.Identity):
return args[0] if args else next(iter(kwargs.values()))
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
"""Whether the last PP rank needs the target input embedding."""
from vllm.distributed.parallel_state import get_pp_group
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.method not in (
"eagle",
"eagle3",
"dflash",
"dspark",
):
return False
pp = get_pp_group()
return pp.world_size > 1 and pp.is_last_rank
def make_layers(
num_hidden_layers: int,
layer_fn: LayerFn,
+12 -2
View File
@@ -67,6 +67,7 @@ from vllm.model_executor.models.utils import (
is_pp_missing_parameter,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.common.ops.sequence_parallel import (
@@ -1270,6 +1271,8 @@ class DeepseekV4DecoderLayer(nn.Module):
class DeepseekV4Model(nn.Module, EagleModelMixin):
supports_aux_hidden_states_over_pp = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1305,7 +1308,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
dtype=torch.int32,
)
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
@@ -1416,6 +1419,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
input_ids = sp_shard(input_ids)
residual, post_mix, res_mix = None, None, None
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
for idx, layer in enumerate(
@@ -1450,7 +1454,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
return IntermediateTensors(
{
"hidden_states": hidden_states,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
@@ -1468,6 +1477,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.hc_eps,
)
hidden_states = self.norm(hidden_states)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+1 -3
View File
@@ -421,9 +421,7 @@ class K3DSparkForCausalLM(nn.Module):
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = K3DSparkModel(
vllm_config=vllm_config,
start_layer_id=target_layer_num,
+26 -33
View File
@@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import (
is_pp_missing_parameter,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.models.common.ops.sequence_parallel import (
@@ -1121,6 +1122,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
"fused_qkv_a_g_proj": ["q_a_proj", "kv_a_proj_with_mqa", "g_proj"],
}
supports_aux_hidden_states_over_pp = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1145,7 +1148,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
vllm_config, self.use_sequence_parallel
)
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
@@ -1228,15 +1231,18 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
super()._set_aux_hidden_state_layers(layers)
if self.use_attn_res and self._aux_attn_res_stream:
pp = get_pp_group()
if not pp.is_last_rank and self.end_layer in self.aux_hidden_state_layers:
raise ValueError(
f"Auxiliary layer {self.end_layer} cannot end a non-final PP "
"stage when VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=1"
)
if self.use_attn_res:
# Emitted once, at configuration time. Which layers are tapped and
# which convention is in force are the two things you need to
# confirm from a running process, and neither is recoverable from
# the served output.
logger.info_once(
"Kimi-K3 aux hidden capture: layers=%s mode=%s "
"(VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=%d)",
layers,
self.aux_hidden_state_layers,
"attn_res_stream" if self._aux_attn_res_stream else "prefix_only",
int(self._aux_attn_res_stream),
)
@@ -1252,25 +1258,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
pending_mlp_out: torch.Tensor | None,
block_residual: torch.Tensor,
) -> torch.Tensor:
"""Auxiliary feature tapped after ``layer_idx`` under AttnRes.
The wire between layers only carries the current block's running prefix;
the committed blocks live in the bank. The value the next consumer
actually reads is the pre-norm AttnRes mixture over
``bank[:num_blocks] + prefix``, which is what the DFlash drafters were
trained against. ``attn_res`` with no delta, no block write and no
output norm computes exactly that and leaves both the prefix and the
bank untouched.
Folding the pending MLP output into the prefix rather than passing it as
``delta`` is deliberate: the kernel writes an applied delta back into
the prefix in place, which would double-add it into the live residual
stream.
"""
"""Return the AttnRes stream after ``layer_idx``."""
prefix = prefix_sum if pending_mlp_out is None else prefix_sum + pending_mlp_out
# `use_attn_res` is what constructs the norm and projection weights this
# reads; without it there is no mixture to compute and the attribute
# lookups below would raise.
if not (self._aux_attn_res_stream and self.use_attn_res):
return prefix
@@ -1280,17 +1269,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
score_proj = consumer.self_attention_res_proj
num_blocks = consumer.prev_valid_blocks
elif get_pp_group().is_last_rank:
# Nothing downstream but the model's own output-side aggregation.
score_norm = self.output_attn_res_norm
score_proj = self.output_attn_res_proj
num_blocks = self.num_attn_res_blocks
else:
# Last layer of a non-final pipeline stage: the consumer lives on
# the next rank and the output-side aggregation only exists on the
# last one, so there is nothing here to mix against. Falling back
# to the running prefix keeps the tap defined rather than reaching
# for weights this rank does not construct.
return prefix
raise RuntimeError("Auxiliary AttnRes capture crossed a PP boundary")
return attn_res(
prefix,
@@ -1338,9 +1321,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
hidden_states = sp_shard(hidden_states)
assert residual is None, "Currently, SP is not supported with PP"
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
# sharded aux hidden states when sp is enabled
aux_hidden_states: list[torch.Tensor] = []
if self.start_layer in self.aux_hidden_state_layers:
if (
get_pp_group().is_first_rank
and self.start_layer in self.aux_hidden_state_layers
):
if self.use_attn_res or residual is None:
aux_hidden_states.append(hidden_states)
else:
@@ -1391,7 +1379,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
if prefix_sum is not None:
hidden_states = hidden_states + prefix_sum
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if self.use_attn_res:
@@ -1428,6 +1420,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
# NOTE: the final norm is applied in compute_logits instead of here, so
# the MTP draft model receives the pre-norm hidden states.
aux_hidden_states = remote_aux + aux_hidden_states
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
+20 -7
View File
@@ -152,6 +152,7 @@ from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
)
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
set_eagle3_aux_hidden_state_layers,
verify_supports_aux_hidden_states_over_pp,
)
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
@@ -280,11 +281,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
):
# Drafting may require auxiliary hidden states from target model outputs
self.use_aux_hidden_state_outputs = True
if self.use_pp:
raise ValueError(
f"{self.speculative_config.method} with pipeline parallel "
"is not supported."
)
# Draft tokens propagation - for spec-dec + struct outputs.
self.draft_tokens_handler = DraftTokensHandler(self.device)
@@ -407,6 +403,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.use_aux_hidden_state_outputs:
assert self.speculative_config is not None
set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config)
if self.use_pp:
assert self.speculative_config.method is not None
verify_supports_aux_hidden_states_over_pp(
self.model, self.speculative_config.method
)
assert self.pp_handler is not None
self.pp_handler.configure_aux_hidden_state_relay(self.model)
if isinstance(self.speculator, DraftModelSpeculator):
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.load_model(self.model)
@@ -1043,7 +1046,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# For non-last PP ranks, update decode requests with sampler output from
# the prior step in which they were scheduled (pp_size steps ago).
if self.pp_handler is not None:
outputs = self.pp_handler.get_prev_sampled_outputs()
outputs = self.pp_handler.get_prev_sampled_outputs(
self.req_states.draft_tokens
)
if outputs is not None:
self.postprocess_sampled(**outputs)
@@ -1879,7 +1884,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not self.is_last_pp_rank:
# Non-last PP rank: return IntermediateTensors for sending.
return output_intermediate_tensors
assert output_intermediate_tensors is not None
assert self.pp_handler is not None
return self.pp_handler.relay_aux_hidden_states(
model_inputs["intermediate_tensors"], output_intermediate_tensors
)
return None
@torch.inference_mode()
@@ -2036,6 +2045,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch,
self.req_states.draft_tokens[input_batch.idx_mapping],
)
if self.pp_handler is not None:
self.pp_handler.broadcast_drafts(
self.req_states.draft_tokens, input_batch
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
+76 -2
View File
@@ -10,6 +10,7 @@ import torch
from vllm.distributed.parallel_state import get_pp_group
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -30,6 +31,7 @@ class PendingRecv:
# Snapshot of slot generation counters at receive time, used to
# detect requests aborted since then.
gen_at_receive_np: np.ndarray # [num_reqs]
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
@@ -61,6 +63,7 @@ class PPHandler:
self.is_last_rank = get_pp_group().is_last_rank
self.last_rank = get_pp_group().last_rank
self.max_sample_len = num_speculative_steps + 1
self.num_speculative_steps = num_speculative_steps
self.device = device
self.main_stream = torch.cuda.current_stream(device)
self.broadcast_stream = torch.cuda.Stream(device)
@@ -82,11 +85,37 @@ class PPHandler:
self.broadcast_group = get_pp_group().make_sibling_device_group(
group_desc="pp_broadcast"
)
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
def on_req_idx_freed(self, req_idx: int) -> None:
self.req_idx_gen_np[req_idx] += 1
def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None:
def configure_aux_hidden_state_relay(self, model: torch.nn.Module) -> None:
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
aux_hidden_state_relay_keys,
)
self.aux_hidden_state_relay_keys = aux_hidden_state_relay_keys(model)
def relay_aux_hidden_states(
self,
intermediate_tensors: IntermediateTensors | None,
output_intermediate_tensors: IntermediateTensors,
) -> IntermediateTensors:
if not self.aux_hidden_state_relay_keys:
return output_intermediate_tensors
assert intermediate_tensors is not None
return IntermediateTensors(
output_intermediate_tensors.tensors
| {
key: intermediate_tensors[key]
for key in self.aux_hidden_state_relay_keys
}
)
def get_prev_sampled_outputs(
self, draft_tokens_to_update: torch.Tensor | None = None
) -> dict[str, torch.Tensor] | None:
"""Consume the entry from pp_size steps ago and wait for its recv event,
then filter out entries whose request was freed since `receive`.
"""
@@ -112,6 +141,18 @@ class PPHandler:
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
self.main_stream.wait_event(slot.event)
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
draft_tokens = slot.draft_tokens
draft_idx_mapping = slot.idx_mapping
if exclude_mask.any():
keep = ~exclude_mask
keep_t = torch.as_tensor(keep, device=self.device)
draft_tokens = draft_tokens[keep_t]
draft_idx_mapping = async_copy_to_gpu(
slot.idx_mapping_np[keep], device=self.device
)
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
return dict(
sampled_tokens=slot.sampled_tokens,
num_sampled=slot.num_sampled,
@@ -119,6 +160,21 @@ class PPHandler:
idx_mapping=idx_mapping,
)
def broadcast_drafts(
self, draft_tokens: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
return
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send = draft_tokens[input_batch.idx_mapping].contiguous()
torch.distributed.broadcast(
send, src=self.last_rank, group=self.broadcast_group
)
send.record_stream(self.broadcast_stream)
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
@@ -145,12 +201,25 @@ class PPHandler:
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
draft_tokens = None
if self.num_speculative_steps > 0:
draft_tokens = torch.empty(
num_reqs,
self.num_speculative_steps,
dtype=torch.int64,
device=self.device,
)
torch.distributed.broadcast(
draft_tokens, src=self.last_rank, group=self.broadcast_group
)
event = self.broadcast_stream.record_event()
num_sampled, num_rejected = combined.unbind(dim=0)
# Must record_stream since these were allocated on broadcast stream but
# later used on the main stream.
sampled_tokens.record_stream(self.main_stream)
combined.record_stream(self.main_stream)
if draft_tokens is not None:
draft_tokens.record_stream(self.main_stream)
self.queue[-1] = PendingRecv(
event,
sampled_tokens,
@@ -160,6 +229,7 @@ class PPHandler:
input_batch.idx_mapping_np,
need_sampled_mask,
gen_at_receive_np,
draft_tokens,
)
return bool(need_sampled_mask.all())
@@ -182,8 +252,12 @@ class PPHandler:
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send_tokens = torch.nn.functional.pad(
sampled_token_ids,
(0, self.max_sample_len - sampled_token_ids.shape[-1]),
)
torch.distributed.broadcast(
sampled_token_ids.contiguous(),
send_tokens.contiguous(),
src=self.last_rank,
group=self.broadcast_group,
)
+2 -13
View File
@@ -3,11 +3,11 @@
import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
maybe_share_target_embed,
)
@@ -54,18 +54,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
target_inner = getattr(target_language_model, "model", target_language_model)
draft_inner = dflash_model.model
# Skip embedding sharing under PP — each rank owns its own embedding.
if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
dflash_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
maybe_share_target_embed(dflash_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
+4 -16
View File
@@ -4,7 +4,6 @@
import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -43,6 +42,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
maybe_share_target_embed,
)
draft_attention_backend = _resolve_dspark_attention_backend(
@@ -53,6 +53,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
draft_vllm_config = replace(
vllm_config,
parallel_config=speculative_config.draft_parallel_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
@@ -76,9 +77,6 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
vllm_config=draft_vllm_config, model_config=draft_model_config
)
if get_pp_group().world_size != 1:
raise NotImplementedError("DSpark does not support pipeline parallelism.")
target_language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
@@ -88,18 +86,8 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
draft_inner = draft_model.model
target_vocab_size = vllm_config.model_config.get_vocab_size()
target_embed = getattr(target_inner, "embed_tokens", None)
draft_embed = getattr(draft_inner, "embed_tokens", None)
if (
target_embed is not None
and draft_model_config.get_vocab_size() <= target_vocab_size
and _should_share(
draft_model, "has_own_embed_tokens", draft_embed, target_embed
)
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
if draft_model_config.get_vocab_size() <= target_vocab_size:
maybe_share_target_embed(draft_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(draft_model, "lm_head", None)
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import torch
import torch.nn as nn
from vllm.config import SpeculativeConfig
@@ -17,8 +18,6 @@ def set_eagle3_aux_hidden_state_layers(
) -> None:
if not supports_eagle3(model):
raise RuntimeError("Model does not support EAGLE3 interface")
# mypy may infer the class-level overload for supports_eagle3.
# Narrow explicitly to the runtime protocol instance.
if isinstance(model, type):
raise RuntimeError("Expected model instance for EAGLE3 configuration")
eagle3_model = cast(SupportsEagle3, model)
@@ -30,6 +29,72 @@ def set_eagle3_aux_hidden_state_layers(
aux_layers = eagle3_model.get_eagle3_default_aux_hidden_state_layers()
logger.info("Using Eagle3 auxiliary layers from model: %s", aux_layers)
eagle3_model.set_aux_hidden_state_layers(aux_layers)
reserve_aux_intermediate_tensor_slots(model)
def _inner_decoder(model: nn.Module) -> nn.Module | None:
parent_ref = model
if hasattr(model, "get_language_model"):
parent_ref = model.get_language_model()
elif hasattr(model, "language_model"):
parent_ref = model.language_model
return getattr(parent_ref, "model", None)
def verify_supports_aux_hidden_states_over_pp(model: nn.Module, method: str) -> None:
inner = _inner_decoder(model)
if not getattr(inner, "supports_aux_hidden_states_over_pp", False):
raise ValueError(
f"{type(model).__name__} does not support {method} with "
"pipeline parallelism"
)
def aux_hidden_state_relay_keys(model: nn.Module) -> tuple[str, ...]:
from vllm.distributed.parallel_state import get_pp_group
pp = get_pp_group()
if pp.world_size < 2 or pp.is_first_rank or pp.is_last_rank:
return ()
inner = _inner_decoder(model)
assert inner is not None
return tuple(
f"{inner.AUX_HIDDEN_STATE_KEY}{i}" for i in range(inner._aux_slot_base_cached)
)
def reserve_aux_intermediate_tensor_slots(model: nn.Module) -> None:
from vllm.distributed.parallel_state import (
get_pp_group,
model_parallel_is_initialized,
)
if not model_parallel_is_initialized():
return
pp = get_pp_group()
if pp.world_size < 2 or pp.is_first_rank:
return
inner = _inner_decoder(model)
if inner is None or not getattr(inner, "supports_aux_hidden_states_over_pp", False):
return
num_aux_states = inner._aux_slot_base_cached
if num_aux_states == 0:
return
key = inner.AUX_HIDDEN_STATE_KEY
hidden_size = inner.config.hidden_size
make_empty = model.make_empty_intermediate_tensors
def make_empty_with_aux(batch_size, dtype, device):
tensors = make_empty(batch_size, dtype, device)
for i in range(num_aux_states):
tensors[f"{key}{i}"] = torch.zeros(
(batch_size, hidden_size), dtype=dtype, device=device
)
return tensors
model.make_empty_intermediate_tensors = make_empty_with_aux
def get_eagle3_aux_layers_from_config(
+37 -19
View File
@@ -7,6 +7,7 @@ from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.models.utils import PPMissingLayer
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
@@ -33,6 +34,41 @@ def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module
)
def maybe_share_target_embed(
draft_model: nn.Module, draft_inner: nn.Module, target_inner: nn.Module
) -> None:
"""Share the target input embedding with the drafter when needed."""
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
if isinstance(target_embed, PPMissingLayer):
target_embed = None
# The drafter does not use the target's LoRA adapter.
if isinstance(target_embed, BaseLayerWithLoRA):
target_embed = target_embed.base_layer
draft_embed = getattr(draft_inner, "embed_tokens", None)
if get_pp_group().world_size > 1 and not hasattr(
draft_model, "has_own_embed_tokens"
):
return
if target_embed is None:
if hasattr(draft_inner, "embed_tokens") and not getattr(
draft_model, "has_own_embed_tokens", False
):
raise RuntimeError(
f"{type(draft_model).__name__} needs the target input embedding, "
"but it is unavailable on this PP stage"
)
return
if _should_share(draft_model, "has_own_embed_tokens", draft_embed, target_embed):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag
@@ -70,25 +106,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
target_inner = target_language_model.model
draft_inner = eagle_model.model
# Skip embedding sharing under PP — each rank owns its own embedding.
if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
)
# If the target's embedding is LoRA-wrapped, share the underlying base
# layer. The draft is not part of the LoRA adapter; sharing the wrapper
# would make the draft run the LoRA embedding kernel with the target's
# punica metadata (sized for the target's token count), causing an
# out-of-bounds GPU access during multi-step draft decode.
if isinstance(target_embed, BaseLayerWithLoRA):
target_embed = target_embed.base_layer
draft_embed = getattr(draft_inner, "embed_tokens", None)
if target_embed is not None and _should_share(
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
):
if draft_embed is not None:
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
maybe_share_target_embed(eagle_model, draft_inner, target_inner)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(eagle_model, "lm_head", None)