[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -67,6 +67,7 @@ from vllm.model_executor.models.utils import (
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
@@ -1270,6 +1271,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
|
||||
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
supports_aux_hidden_states_over_pp = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -1305,7 +1308,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
@@ -1416,6 +1419,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
input_ids = sp_shard(input_ids)
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
|
||||
for idx, layer in enumerate(
|
||||
@@ -1450,7 +1454,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
return IntermediateTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
**self.pack_local_aux_hidden_states(aux_hidden_states),
|
||||
}
|
||||
)
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
@@ -1468,6 +1477,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.hc_eps,
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
aux_hidden_states = remote_aux + aux_hidden_states
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -421,9 +421,7 @@ class K3DSparkForCausalLM(nn.Module):
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
target_layer_num = vllm_config.model_config.get_total_num_hidden_layers()
|
||||
self.model = K3DSparkModel(
|
||||
vllm_config=vllm_config,
|
||||
start_layer_id=target_layer_num,
|
||||
|
||||
@@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import (
|
||||
is_pp_missing_parameter,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
@@ -1121,6 +1122,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
"fused_qkv_a_g_proj": ["q_a_proj", "kv_a_proj_with_mqa", "g_proj"],
|
||||
}
|
||||
|
||||
supports_aux_hidden_states_over_pp = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -1145,7 +1148,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
vllm_config, self.use_sequence_parallel
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if get_pp_group().is_first_rank or spec_decode_needs_target_embed(vllm_config):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
@@ -1228,15 +1231,18 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
|
||||
def _set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
super()._set_aux_hidden_state_layers(layers)
|
||||
if self.use_attn_res and self._aux_attn_res_stream:
|
||||
pp = get_pp_group()
|
||||
if not pp.is_last_rank and self.end_layer in self.aux_hidden_state_layers:
|
||||
raise ValueError(
|
||||
f"Auxiliary layer {self.end_layer} cannot end a non-final PP "
|
||||
"stage when VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=1"
|
||||
)
|
||||
if self.use_attn_res:
|
||||
# Emitted once, at configuration time. Which layers are tapped and
|
||||
# which convention is in force are the two things you need to
|
||||
# confirm from a running process, and neither is recoverable from
|
||||
# the served output.
|
||||
logger.info_once(
|
||||
"Kimi-K3 aux hidden capture: layers=%s mode=%s "
|
||||
"(VLLM_KIMI_K3_AUX_ATTN_RES_STREAM=%d)",
|
||||
layers,
|
||||
self.aux_hidden_state_layers,
|
||||
"attn_res_stream" if self._aux_attn_res_stream else "prefix_only",
|
||||
int(self._aux_attn_res_stream),
|
||||
)
|
||||
@@ -1252,25 +1258,8 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
pending_mlp_out: torch.Tensor | None,
|
||||
block_residual: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Auxiliary feature tapped after ``layer_idx`` under AttnRes.
|
||||
|
||||
The wire between layers only carries the current block's running prefix;
|
||||
the committed blocks live in the bank. The value the next consumer
|
||||
actually reads is the pre-norm AttnRes mixture over
|
||||
``bank[:num_blocks] + prefix``, which is what the DFlash drafters were
|
||||
trained against. ``attn_res`` with no delta, no block write and no
|
||||
output norm computes exactly that and leaves both the prefix and the
|
||||
bank untouched.
|
||||
|
||||
Folding the pending MLP output into the prefix rather than passing it as
|
||||
``delta`` is deliberate: the kernel writes an applied delta back into
|
||||
the prefix in place, which would double-add it into the live residual
|
||||
stream.
|
||||
"""
|
||||
"""Return the AttnRes stream after ``layer_idx``."""
|
||||
prefix = prefix_sum if pending_mlp_out is None else prefix_sum + pending_mlp_out
|
||||
# `use_attn_res` is what constructs the norm and projection weights this
|
||||
# reads; without it there is no mixture to compute and the attribute
|
||||
# lookups below would raise.
|
||||
if not (self._aux_attn_res_stream and self.use_attn_res):
|
||||
return prefix
|
||||
|
||||
@@ -1280,17 +1269,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
score_proj = consumer.self_attention_res_proj
|
||||
num_blocks = consumer.prev_valid_blocks
|
||||
elif get_pp_group().is_last_rank:
|
||||
# Nothing downstream but the model's own output-side aggregation.
|
||||
score_norm = self.output_attn_res_norm
|
||||
score_proj = self.output_attn_res_proj
|
||||
num_blocks = self.num_attn_res_blocks
|
||||
else:
|
||||
# Last layer of a non-final pipeline stage: the consumer lives on
|
||||
# the next rank and the output-side aggregation only exists on the
|
||||
# last one, so there is nothing here to mix against. Falling back
|
||||
# to the running prefix keeps the tap defined rather than reaching
|
||||
# for weights this rank does not construct.
|
||||
return prefix
|
||||
raise RuntimeError("Auxiliary AttnRes capture crossed a PP boundary")
|
||||
|
||||
return attn_res(
|
||||
prefix,
|
||||
@@ -1338,9 +1321,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
assert residual is None, "Currently, SP is not supported with PP"
|
||||
|
||||
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
|
||||
|
||||
# sharded aux hidden states when sp is enabled
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if self.start_layer in self.aux_hidden_state_layers:
|
||||
if (
|
||||
get_pp_group().is_first_rank
|
||||
and self.start_layer in self.aux_hidden_state_layers
|
||||
):
|
||||
if self.use_attn_res or residual is None:
|
||||
aux_hidden_states.append(hidden_states)
|
||||
else:
|
||||
@@ -1391,7 +1379,11 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
if prefix_sum is not None:
|
||||
hidden_states = hidden_states + prefix_sum
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
**self.pack_local_aux_hidden_states(aux_hidden_states),
|
||||
}
|
||||
)
|
||||
|
||||
if self.use_attn_res:
|
||||
@@ -1428,6 +1420,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
|
||||
# NOTE: the final norm is applied in compute_logits instead of here, so
|
||||
# the MTP draft model receives the pre-norm hidden states.
|
||||
aux_hidden_states = remote_aux + aux_hidden_states
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
Reference in New Issue
Block a user