[PCP][Spec Decode] Adds PCP support for single-module MTP and replicated DSpark. (#56107)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Summer Yang
OpenAI Codex
OpenAI Codex
parent
e7edf17cea
commit
980c16c8e4
@@ -250,8 +250,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
|
||||
|
||||
self._copy_request_inputs(
|
||||
num_reqs,
|
||||
input_batch.idx_mapping,
|
||||
@@ -272,6 +270,17 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
self.max_num_reqs,
|
||||
)
|
||||
|
||||
if self.pcp_manager is not None:
|
||||
self.pcp_manager.prepare_draft_prefill(
|
||||
input_batch,
|
||||
self.input_buffers.input_ids[:num_tokens_padded],
|
||||
)
|
||||
prefill = self.pcp_manager.draft_prefill_batch
|
||||
if prefill is not None:
|
||||
num_tokens = prefill.num_tokens
|
||||
num_tokens_padded = prefill.num_tokens_after_padding
|
||||
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
|
||||
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_decode_token_count(
|
||||
@@ -324,6 +333,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
# Early exit.
|
||||
return self.draft_tokens[:num_reqs, :1]
|
||||
|
||||
if self.pcp_manager is not None and not dummy_run:
|
||||
self.block_tables.gather_block_tables(
|
||||
input_batch.idx_mapping, num_reqs_padded=num_reqs
|
||||
)
|
||||
|
||||
# Prepare the inputs for the decode steps.
|
||||
prepare_decode_inputs(
|
||||
self.draft_tokens[:num_reqs, 0],
|
||||
@@ -387,6 +401,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
input_buffers: InputBatch | InputBuffers = self.input_buffers
|
||||
is_padding = None
|
||||
if self.pcp_manager is not None:
|
||||
input_buffers = self.pcp_manager.get_draft_input_buffers(self.input_buffers)
|
||||
is_padding = input_buffers.is_padding[:num_tokens]
|
||||
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
|
||||
with set_forward_context(
|
||||
attn_metadata,
|
||||
@@ -396,6 +415,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
is_padding=is_padding,
|
||||
):
|
||||
inputs_embeds = None
|
||||
if self.supports_mm_inputs:
|
||||
@@ -406,15 +426,15 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
is_mm_embed.shape[0] if is_mm_embed is not None else num_tokens
|
||||
)
|
||||
self.inputs_embeds[:num_input_tokens] = self.model.embed_input_ids(
|
||||
self.input_buffers.input_ids[:num_input_tokens],
|
||||
input_buffers.input_ids[:num_input_tokens],
|
||||
multimodal_embeddings=mm_embeds,
|
||||
is_multimodal=is_mm_embed,
|
||||
)
|
||||
inputs_embeds = self.inputs_embeds[:num_tokens]
|
||||
|
||||
model_inputs = dict(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
positions=self.input_buffers.positions[:num_tokens],
|
||||
input_ids=input_buffers.input_ids[:num_tokens],
|
||||
positions=input_buffers.positions[:num_tokens],
|
||||
hidden_states=self.hidden_states[:num_tokens],
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
@@ -463,6 +483,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
if self.pcp_manager is not None:
|
||||
last_hidden_states, hidden_states = self.pcp_manager.restore_draft_prefill(
|
||||
last_hidden_states, hidden_states
|
||||
)
|
||||
|
||||
sample_hidden_states = last_hidden_states[last_token_indices]
|
||||
self.draft_tokens[:num_reqs, 0] = self.sample_draft(
|
||||
|
||||
@@ -38,6 +38,13 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
_speculator_name = "DFlash" # For logging, so we can share methods with subclasses
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if parallel_config.prefill_context_parallel_size > 1:
|
||||
vllm_config = copy.copy(vllm_config)
|
||||
vllm_config.parallel_config = replace(
|
||||
parallel_config,
|
||||
prefill_context_parallel_size=1,
|
||||
)
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
self.hidden_states = torch.zeros(
|
||||
@@ -388,6 +395,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
if self.pcp_manager is not None and not dummy_run:
|
||||
self.block_tables.gather_block_tables(
|
||||
input_batch.idx_mapping, num_reqs_padded=num_reqs
|
||||
)
|
||||
|
||||
# The query slot mapping is written into the shared BlockTables slot_mappings.
|
||||
# That buffer's address is what the captured CUDA graph reads from at replay.
|
||||
assert self.draft_kv_cache_group_id >= 0
|
||||
|
||||
@@ -28,7 +28,8 @@ class MTPSpeculator(AutoRegressiveSpeculator):
|
||||
# toggles skip_topk so step 0 computes MTP's own indices and
|
||||
# steps 1+ reuse them.
|
||||
self.share_mtp_topk_indices = (
|
||||
getattr(draft_hf_config, "index_share_for_mtp_iteration", False)
|
||||
self.vllm_config.parallel_config.prefill_context_parallel_size == 1
|
||||
and getattr(draft_hf_config, "index_share_for_mtp_iteration", False)
|
||||
and hasattr(draft_model.model, "set_skip_topk")
|
||||
and hasattr(draft_model.model, "compact_topk_indices")
|
||||
)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -29,6 +29,9 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -149,6 +152,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.arange = torch.arange(
|
||||
self.max_num_reqs + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
@@ -166,6 +170,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
)
|
||||
|
||||
self.supports_mm_inputs = False
|
||||
self.pcp_manager: PCPManager | None = None
|
||||
|
||||
@abstractmethod
|
||||
def load_draft_model(
|
||||
@@ -326,6 +331,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
causal=causal,
|
||||
seq_lens_cpu_upper_bound=draft_seq_lens_cpu_upper_bound,
|
||||
is_prefilling=self.draft_is_prefilling[:num_reqs],
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
|
||||
Reference in New Issue
Block a user