[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chris Eastwood
2026-09-11 17:55:21 +00:00
committed by GitHub
co-authored by Nick Hill mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9a35c081e8
commit 6fe67cbbf3
12 changed files with 167 additions and 49 deletions
+9 -3
View File
@@ -38,6 +38,7 @@ from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
_require_is_multimodal,
)
from .utils import (
@@ -152,7 +153,9 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
# Branch on the inputs, not the rank: the drafter is built entirely on
# the last PP stage, where `is_first_rank` is False.
if intermediate_tensors is None:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
assert hidden_states.shape[-1] == inputs_embeds.shape[-1]
@@ -162,7 +165,6 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
hidden_states = self.fc(hidden_states)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
@@ -212,7 +214,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
"hidden_states": 0,
}
)
class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal):
class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -254,6 +256,10 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal):
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(
self,
input_ids: torch.Tensor,