[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
+32 -7
View File
@@ -48,7 +48,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import MixtureOfExperts, SupportsPP
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
MixtureOfExperts,
SupportsEagle3,
SupportsPP,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
@@ -933,7 +938,7 @@ class DeepseekV4DecoderLayer(nn.Module):
return x, residual, post_mix, res_mix
class DeepseekV4Model(nn.Module):
class DeepseekV4Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1074,7 +1079,12 @@ class DeepseekV4Model(nn.Module):
input_ids = input_ids.to(torch.int64)
residual, post_mix, res_mix = None, None, None
for layer in islice(self.layers, self.start_layer, self.end_layer):
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual, post_mix, res_mix = layer(
hidden_states,
positions,
@@ -1083,10 +1093,21 @@ class DeepseekV4Model(nn.Module):
res_mix,
residual,
)
if idx + 1 in self.aux_hidden_state_layers:
# Reconstruct the aux hidden state for draft models
aux_recon = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
aux_hidden_states.append(aux_recon.mean(dim=1))
final_aux_recon = aux_recon
if layer is not None:
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
# Reuse if the last layer was captured as an aux hidden state
if self.end_layer in self.aux_hidden_state_layers:
hidden_states = final_aux_recon
else:
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
@@ -1104,6 +1125,8 @@ class DeepseekV4Model(nn.Module):
self.hc_eps,
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -1330,7 +1353,9 @@ class DeepseekV4MixtureOfExperts(MixtureOfExperts):
moe.experts.update_expert_map()
class DeepseekV4ForCausalLM(nn.Module, SupportsPP, DeepseekV4MixtureOfExperts):
class DeepseekV4ForCausalLM(
nn.Module, SupportsPP, SupportsEagle3, DeepseekV4MixtureOfExperts
):
model_cls = DeepseekV4Model
# Default mapper assumes the original FP4-expert checkpoint layout.