[Bugfix][Spec Decode] Implement get_top_tokens() on the ROCm DeepSeek V4 MTP drafter (#57568)
Signed-off-by: BaoYunkai <ybao@amd.com>
This commit is contained in:
@@ -240,13 +240,11 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
def _pre_head(
|
||||
self,
|
||||
mtp_layer: DeepSeekV4MultiTokenPredictorLayer,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply
|
||||
# hc_head here so logits are computed from the dense hidden state.
|
||||
hidden_states = hidden_states.view(
|
||||
@@ -260,14 +258,38 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
return _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
)
|
||||
logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
logits = self.logits_processor(
|
||||
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
|
||||
)
|
||||
return logits
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
# Vocab-parallel argmax for the greedy draft: per-rank head projection
|
||||
# + local argmax + a [batch, 2*tp] (value, index) reduce, instead of
|
||||
# all-gathering the full vocab logits.
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states)
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV4MTP(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
@@ -302,6 +324,15 @@ class DeepSeekV4MTP(nn.Module):
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Greedy-draft path used when use_local_argmax_reduction is enabled:
|
||||
# vocab-parallel argmax, no full-vocab logits.
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Weight name remapping for checkpoint compatibility.
|
||||
# Maps checkpoint weight paths to model parameter paths.
|
||||
|
||||
Reference in New Issue
Block a user