[Model Runner V2][Spec Decode] Skip DP sync before EAGLE/MTP draft prefill (#53694)

This commit is contained in:
Giancarlo Delfin
2026-08-26 20:59:35 -07:00
committed by GitHub
parent ca90b9e7d4
commit 5acc1c4e4b
8 changed files with 156 additions and 36 deletions
@@ -14,7 +14,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
@@ -219,7 +219,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -279,7 +279,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
max_query_len,
input_batch.has_prefill,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
prefill_batch_desc, prefill_batch_sync = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens_padded,
@@ -287,6 +287,12 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
)
num_tokens_across_dp = (
prefill_batch_sync.num_tokens_across_dp
if prefill_batch_sync is not None
else None
)
self._prepare_eplb_forward(num_tokens)
@@ -328,7 +334,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# Each request produces exactly 1 token per draft generation step,
# enabling FULL graph replay.
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
self.decode_cudagraph_manager,
num_reqs,
num_reqs,
@@ -337,6 +343,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
dp_rank=self.dp_rank,
need_eager=is_profile,
)
num_tokens_across_dp = (
decode_batch_sync.num_tokens_across_dp
if decode_batch_sync is not None
else None
)
self.on_multi_step_decode_begin(num_reqs)
# Generate the remaining num_speculative_steps - 1 draft tokens.
@@ -19,7 +19,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
@@ -335,7 +335,7 @@ class DFlashSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -377,7 +377,9 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_tokens,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=num_tokens_across_dp,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
return self.draft_tokens[:num_reqs]
@@ -438,7 +440,7 @@ class DFlashSpeculator(DraftModelSpeculator):
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.query_cudagraph_manager,
num_reqs,
num_query_tokens,
@@ -450,6 +452,9 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (
batch_sync.num_tokens_across_dp if batch_sync is not None else None
)
# Rebuild the draft attention metadata even when replaying the FULL
# graph so that any attention metadata builder state is updated.
@@ -10,6 +10,7 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import set_forward_context
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@@ -97,7 +98,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
next_prefill_tokens: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -143,7 +144,9 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
draft_attn_metadata,
self.vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=num_tokens_across_dp,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
cudagraph_runtime_mode=CUDAGraphMode.NONE,
slot_mapping=draft_slot_mappings,
is_padding=input_batch.is_padding[:num_tokens],
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
@@ -150,7 +150,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
@@ -191,7 +191,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
uniform_token_count = get_uniform_decode_token_count(
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
)
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.cudagraph_manager,
num_reqs,
input_batch.num_tokens_after_padding,
@@ -199,6 +199,10 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=dp_sync,
)
num_tokens_across_dp = (
batch_sync.num_tokens_across_dp if batch_sync is not None else None
)
# Rebuild the slot mappings and attention metadata.
+2 -1
View File
@@ -21,6 +21,7 @@ from vllm.v1.worker.gpu.attn_utils import (
init_attn_backend,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
@@ -60,7 +61,7 @@ class BaseSpeculator(ABC):
temperature: torch.Tensor,
# [max_num_reqs]
seeds: torch.Tensor,
num_tokens_across_dp: torch.Tensor | None = None,
dp_sync: DPSyncState | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,