[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -48,7 +48,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts, SupportsPP
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -933,7 +938,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
return x, residual, post_mix, res_mix
|
||||
|
||||
|
||||
class DeepseekV4Model(nn.Module):
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -1074,7 +1079,12 @@ class DeepseekV4Model(nn.Module):
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual, post_mix, res_mix = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
@@ -1083,10 +1093,21 @@ class DeepseekV4Model(nn.Module):
|
||||
res_mix,
|
||||
residual,
|
||||
)
|
||||
if idx + 1 in self.aux_hidden_state_layers:
|
||||
# Reconstruct the aux hidden state for draft models
|
||||
aux_recon = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
aux_hidden_states.append(aux_recon.mean(dim=1))
|
||||
final_aux_recon = aux_recon
|
||||
if layer is not None:
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
# Reuse if the last layer was captured as an aux hidden state
|
||||
if self.end_layer in self.aux_hidden_state_layers:
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
@@ -1104,6 +1125,8 @@ class DeepseekV4Model(nn.Module):
|
||||
self.hc_eps,
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -1330,7 +1353,9 @@ class DeepseekV4MixtureOfExperts(MixtureOfExperts):
|
||||
moe.experts.update_expert_map()
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, DeepseekV4MixtureOfExperts):
|
||||
class DeepseekV4ForCausalLM(
|
||||
nn.Module, SupportsPP, SupportsEagle3, DeepseekV4MixtureOfExperts
|
||||
):
|
||||
model_cls = DeepseekV4Model
|
||||
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
|
||||
Reference in New Issue
Block a user