[Bugfix][Spec Decode] Per-module LM heads for multi-layer MTP on Model Runner V2 (#58921)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: chanh <channguyen@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
chanh
Claude Opus 5.5
parent
e5e38ba9b7
commit
c4fc3c7d32
@@ -323,7 +323,11 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
logits = logits[..., : self.logits_processor.org_vocab_size]
|
||||
return logits
|
||||
|
||||
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
|
||||
@@ -323,7 +323,11 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
logits = logits[..., : self.logits_processor.org_vocab_size]
|
||||
return logits
|
||||
|
||||
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Greedy draft tokens via rank-local argmax + tiny (value, index)
|
||||
reduction — no full-vocab logits all-gather. The muP divisor is a
|
||||
positive scalar, so the argmax is invariant and the scaling is
|
||||
|
||||
Reference in New Issue
Block a user