[Bugfix][Spec Decode] Per-module LM heads for multi-layer MTP on Model Runner V2 (#58921)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: chanh <channguyen@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-10-01 01:09:14 -07:00
committed by GitHub
co-authored by chanh Claude Opus 5.5
parent e5e38ba9b7
commit c4fc3c7d32
6 changed files with 60 additions and 10 deletions
@@ -13,10 +13,14 @@ from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
"""Share when the draft has no own copy, or its copy matches the target."""
if not getattr(eagle, flag, False) or draft is None:
"""Share unless the draft declares its own copy that differs from the target.
A draft that declares its own copy but has no top-level one (e.g. MTP
heads stored per layer) keeps it.
"""
if not getattr(eagle, flag, False):
return True
if target is None:
if draft is None or target is None:
return False
# torch.equal on GPU allocates a bool mask the size of the input.
# Use the faster GPU path when there is plenty of headroom;
@@ -253,6 +253,17 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
)
return self.draft_tokens[:num_reqs]
# Each MTP module may carry its own LM head, selected by spec_step_idx.
def compute_draft_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int
) -> torch.Tensor:
return self.model.compute_logits(hidden_states, spec_step_idx=spec_step_idx)
def get_draft_top_tokens(
self, hidden_states: torch.Tensor, spec_step_idx: int
) -> torch.Tensor:
return self.model.get_top_tokens(hidden_states, spec_step_idx=spec_step_idx)
@torch.inference_mode()
def _run_model(
self,
@@ -427,6 +438,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
self.seeds,
self.current_draft_step,
self.draft_logits,
spec_step_idx=step,
)
self.draft_tokens[:num_reqs, step] = draft_tokens
+15 -4
View File
@@ -403,6 +403,16 @@ class DraftModelSpeculator(BaseSpeculator):
"(communication: O(2*tp_size) vs O(vocab_size))."
)
def compute_draft_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int
) -> torch.Tensor:
return self.model.compute_logits(hidden_states)
def get_draft_top_tokens(
self, hidden_states: torch.Tensor, spec_step_idx: int
) -> torch.Tensor:
return self.model.get_top_tokens(hidden_states)
def sample_draft(
self,
hidden_states: torch.Tensor,
@@ -412,9 +422,13 @@ class DraftModelSpeculator(BaseSpeculator):
seeds: torch.Tensor,
draft_step: torch.Tensor,
draft_logits: torch.Tensor | None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if draft_logits is None and self.use_local_argmax_reduction:
return self.get_draft_top_tokens(hidden_states, spec_step_idx)
logits = self.compute_draft_logits(hidden_states, spec_step_idx)
if draft_logits is not None:
logits = self.model.compute_logits(hidden_states)
sampler = (
gumbel_sample
if self.draft_watermarker is None
@@ -432,10 +446,7 @@ class DraftModelSpeculator(BaseSpeculator):
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
elif self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
else:
logits = self.model.compute_logits(hidden_states)
sampled = logits.argmax(dim=-1)
self._maybe_predict_acceptance(logits, idx_mapping, draft_step)
return sampled