[PCP][Spec Decode] Adds PCP support for single-module MTP and replicated DSpark. (#56107)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-10 22:07:53 -07:00
committed by GitHub
co-authored by Summer Yang OpenAI Codex OpenAI Codex
parent e7edf17cea
commit 980c16c8e4
11 changed files with 159 additions and 79 deletions
@@ -250,8 +250,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
self._copy_request_inputs(
num_reqs,
input_batch.idx_mapping,
@@ -272,6 +270,17 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.max_num_reqs,
)
if self.pcp_manager is not None:
self.pcp_manager.prepare_draft_prefill(
input_batch,
self.input_buffers.input_ids[:num_tokens_padded],
)
prefill = self.pcp_manager.draft_prefill_batch
if prefill is not None:
num_tokens = prefill.num_tokens
num_tokens_padded = prefill.num_tokens_after_padding
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_decode_token_count(
@@ -324,6 +333,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# Early exit.
return self.draft_tokens[:num_reqs, :1]
if self.pcp_manager is not None and not dummy_run:
self.block_tables.gather_block_tables(
input_batch.idx_mapping, num_reqs_padded=num_reqs
)
# Prepare the inputs for the decode steps.
prepare_decode_inputs(
self.draft_tokens[:num_reqs, 0],
@@ -387,6 +401,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
input_buffers: InputBatch | InputBuffers = self.input_buffers
is_padding = None
if self.pcp_manager is not None:
input_buffers = self.pcp_manager.get_draft_input_buffers(self.input_buffers)
is_padding = input_buffers.is_padding[:num_tokens]
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata,
@@ -396,6 +415,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=is_padding,
):
inputs_embeds = None
if self.supports_mm_inputs:
@@ -406,15 +426,15 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
)
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
self.input_buffers.input_ids[:num_input_tokens],
input_buffers.input_ids[:num_input_tokens],
multimodal_embeddings=mm_embeds,
is_multimodal=is_mm_embed,
)
inputs_embeds = self.inputs_embeds[:num_tokens]
model_inputs = dict(
input_ids=self.input_buffers.input_ids[:num_tokens],
positions=self.input_buffers.positions[:num_tokens],
input_ids=input_buffers.input_ids[:num_tokens],
positions=input_buffers.positions[:num_tokens],
hidden_states=self.hidden_states[:num_tokens],
inputs_embeds=inputs_embeds,
)
@@ -463,6 +483,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_runtime_mode=cudagraph_runtime_mode,
mm_inputs=mm_inputs,
)
if self.pcp_manager is not None:
last_hidden_states, hidden_states = self.pcp_manager.restore_draft_prefill(
last_hidden_states, hidden_states
)
sample_hidden_states = last_hidden_states[last_token_indices]
self.draft_tokens[:num_reqs, 0] = self.sample_draft(
@@ -38,6 +38,13 @@ class DFlashSpeculator(DraftModelSpeculator):
_speculator_name = "DFlash" # For logging, so we can share methods with subclasses
def __init__(self, vllm_config: VllmConfig, device: torch.device):
parallel_config = vllm_config.parallel_config
if parallel_config.prefill_context_parallel_size > 1:
vllm_config = copy.copy(vllm_config)
vllm_config.parallel_config = replace(
parallel_config,
prefill_context_parallel_size=1,
)
super().__init__(vllm_config, device)
self.hidden_states = torch.zeros(
@@ -388,6 +395,11 @@ class DFlashSpeculator(DraftModelSpeculator):
)
return self.draft_tokens[:num_reqs]
if self.pcp_manager is not None and not dummy_run:
self.block_tables.gather_block_tables(
input_batch.idx_mapping, num_reqs_padded=num_reqs
)
# The query slot mapping is written into the shared BlockTables slot_mappings.
# That buffer's address is what the captured CUDA graph reads from at replay.
assert self.draft_kv_cache_group_id >= 0
@@ -28,7 +28,8 @@ class MTPSpeculator(AutoRegressiveSpeculator):
# toggles skip_topk so step 0 computes MTP's own indices and
# steps 1+ reuse them.
self.share_mtp_topk_indices = (
getattr(draft_hf_config, "index_share_for_mtp_iteration", False)
self.vllm_config.parallel_config.prefill_context_parallel_size == 1
and getattr(draft_hf_config, "index_share_for_mtp_iteration", False)
and hasattr(draft_model.model, "set_skip_topk")
and hasattr(draft_model.model, "compact_topk_indices")
)
+7 -1
View File
@@ -3,7 +3,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import replace
from typing import Any
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
@@ -29,6 +29,9 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.pcp_manager import PCPManager
logger = init_logger(__name__)
@@ -149,6 +152,7 @@ class DraftModelSpeculator(BaseSpeculator):
self.arange = torch.arange(
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
)
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "probabilistic":
@@ -166,6 +170,7 @@ class DraftModelSpeculator(BaseSpeculator):
)
self.supports_mm_inputs = False
self.pcp_manager: PCPManager | None = None
@abstractmethod
def load_draft_model(
@@ -326,6 +331,7 @@ class DraftModelSpeculator(BaseSpeculator):
kv_cache_config=self.kv_cache_config,
causal=causal,
seq_lens_cpu_upper_bound=draft_seq_lens_cpu_upper_bound,
is_prefilling=self.draft_is_prefilling[:num_reqs],
)
return attn_metadata