[Bugfix][Spec Decode] Implement get_top_tokens() on the ROCm DeepSeek V4 MTP drafter (#57568)

Signed-off-by: BaoYunkai <ybao@amd.com>
This commit is contained in:
BaoYunkai
2026-09-28 20:16:38 +08:00
committed by GitHub
parent 8cc9aa5ad3
commit f6877f563c
2 changed files with 200 additions and 6 deletions
+37 -6
View File
@@ -240,13 +240,11 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
current_step_idx,
)
def compute_logits(
def _pre_head(
self,
mtp_layer: DeepSeekV4MultiTokenPredictorLayer,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
# hc_head here so logits are computed from the dense hidden state.
hidden_states = hidden_states.view(
@@ -260,14 +258,38 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
return _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
)
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
logits = self.logits_processor(
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
)
return logits
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
# Vocab-parallel argmax for the greedy draft: per-rank head projection
# + local argmax + a [batch, 2*tp] (value, index) reduce, instead of
# all-gathering the full vocab logits.
return self.logits_processor.get_top_tokens(
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
)
class DeepSeekV4MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -302,6 +324,15 @@ class DeepSeekV4MTP(nn.Module):
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
# Greedy-draft path used when use_local_argmax_reduction is enabled:
# vocab-parallel argmax, no full-vocab logits.
return self.model.get_top_tokens(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Weight name remapping for checkpoint compatibility.
# Maps checkpoint weight paths to model parameter paths.