[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+12 -2
View File
@@ -67,6 +67,7 @@ from vllm.model_executor.models.utils import (
is_pp_missing_parameter,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.common.ops.sequence_parallel import (
@@ -1270,6 +1271,8 @@ class DeepseekV4DecoderLayer(nn.Module):
class DeepseekV4Model(nn.Module, EagleModelMixin):
supports_aux_hidden_states_over_pp = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1305,7 +1308,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
dtype=torch.int32,
)
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
@@ -1416,6 +1419,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
input_ids = sp_shard(input_ids)
residual, post_mix, res_mix = None, None, None
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
for idx, layer in enumerate(
@@ -1450,7 +1454,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
return IntermediateTensors(
{
"hidden_states": hidden_states,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
@@ -1468,6 +1477,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.hc_eps,
)
hidden_states = self.norm(hidden_states)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+1 -3
View File
@@ -421,9 +421,7 @@ class K3DSparkForCausalLM(nn.Module):
assert vllm_config.speculative_config is not None
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config
)
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
self.model = K3DSparkModel(
vllm_config=vllm_config,
start_layer_id=target_layer_num,
+26 -33
View File
@@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import (
is_pp_missing_parameter,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.models.common.ops.sequence_parallel import (
@@ -1121,6 +1122,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
"fused_qkv_a_g_proj": ["q_a_proj", "kv_a_proj_with_mqa", "g_proj"],
}
supports_aux_hidden_states_over_pp = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1145,7 +1148,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
vllm_config, self.use_sequence_parallel
)
if get_pp_group().is_first_rank:
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
@@ -1228,15 +1231,18 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
super()._set_aux_hidden_state_layers(layers)
if self.use_attn_res and self._aux_attn_res_stream:
pp = get_pp_group()
if not pp.is_last_rank and self.end_layer in self.aux_hidden_state_layers:
raise ValueError(
f"Auxiliary layer {self.end_layer} cannot end a non-final PP "
"stage when VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=1"
)
if self.use_attn_res:
# Emitted once, at configuration time. Which layers are tapped and
# which convention is in force are the two things you need to
# confirm from a running process, and neither is recoverable from
# the served output.
logger.info_once(
"Kimi-K3 aux hidden capture: layers=%s mode=%s "
"(VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=%d)",
layers,
self.aux_hidden_state_layers,
"attn_res_stream" if self._aux_attn_res_stream else "prefix_only",
int(self._aux_attn_res_stream),
)
@@ -1252,25 +1258,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
pending_mlp_out: torch.Tensor | None,
block_residual: torch.Tensor,
) -> torch.Tensor:
"""Auxiliary feature tapped after ``layer_idx`` under AttnRes.
The wire between layers only carries the current block's running prefix;
the committed blocks live in the bank. The value the next consumer
actually reads is the pre-norm AttnRes mixture over
``bank[:num_blocks] + prefix``, which is what the DFlash drafters were
trained against. ``attn_res`` with no delta, no block write and no
output norm computes exactly that and leaves both the prefix and the
bank untouched.
Folding the pending MLP output into the prefix rather than passing it as
``delta`` is deliberate: the kernel writes an applied delta back into
the prefix in place, which would double-add it into the live residual
stream.
"""
"""Return the AttnRes stream after ``layer_idx``."""
prefix = prefix_sum if pending_mlp_out is None else prefix_sum + pending_mlp_out
# `use_attn_res` is what constructs the norm and projection weights this
# reads; without it there is no mixture to compute and the attribute
# lookups below would raise.
if not (self._aux_attn_res_stream and self.use_attn_res):
return prefix
@@ -1280,17 +1269,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
score_proj = consumer.self_attention_res_proj
num_blocks = consumer.prev_valid_blocks
elif get_pp_group().is_last_rank:
# Nothing downstream but the model's own output-side aggregation.
score_norm = self.output_attn_res_norm
score_proj = self.output_attn_res_proj
num_blocks = self.num_attn_res_blocks
else:
# Last layer of a non-final pipeline stage: the consumer lives on
# the next rank and the output-side aggregation only exists on the
# last one, so there is nothing here to mix against. Falling back
# to the running prefix keeps the tap defined rather than reaching
# for weights this rank does not construct.
return prefix
raise RuntimeError("Auxiliary AttnRes capture crossed a PP boundary")
return attn_res(
prefix,
@@ -1338,9 +1321,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
hidden_states = sp_shard(hidden_states)
assert residual is None, "Currently, SP is not supported with PP"
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
# sharded aux hidden states when sp is enabled
aux_hidden_states: list[torch.Tensor] = []
if self.start_layer in self.aux_hidden_state_layers:
if (
get_pp_group().is_first_rank
and self.start_layer in self.aux_hidden_state_layers
):
if self.use_attn_res or residual is None:
aux_hidden_states.append(hidden_states)
else:
@@ -1391,7 +1379,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
if prefix_sum is not None:
hidden_states = hidden_states + prefix_sum
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if self.use_attn_res:
@@ -1428,6 +1420,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
# NOTE: the final norm is applied in compute_logits instead of here, so
# the MTP draft model receives the pre-norm hidden states.
aux_hidden_states = remote_aux + aux_hidden_states
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states