[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)

This commit is contained in:
Giancarlo Delfin
2026-09-03 10:32:33 +00:00
committed by GitHub
parent 0d3ede3e3b
commit facd9a74a1
4 changed files with 48 additions and 7 deletions
@@ -336,16 +336,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
advance_draft_positions=self.advance_draft_positions,
)
decode_batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
if dp_sync is not None
else (None, num_reqs)
)
# Each request produces exactly 1 token per draft generation step,
# enabling FULL graph replay.
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
self.decode_cudagraph_manager,
num_reqs,
num_reqs,
num_batch_tokens,
uniform_token_count=1,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=decode_batch_sync,
)
num_tokens_across_dp = (
decode_batch_sync.num_tokens_across_dp
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_tokens,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
return self.draft_tokens[:num_reqs]
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
context_slots,
)
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.query_cudagraph_manager,
num_reqs,
num_query_tokens,
num_batch_tokens,
uniform_token_count=self.num_query_per_req,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=batch_sync,
)
num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import replace
from typing import Any
import numpy as np
@@ -409,3 +410,22 @@ class DraftModelSpeculator(BaseSpeculator):
# idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1)
def _build_uniform_batch_dp_sync(
self,
target_dp_sync: DPSyncState,
num_reqs: int,
num_query_per_req: int = 1,
) -> tuple[DPSyncState, int]:
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
assert num_reqs * num_query_per_req <= num_batch_tokens, (
"reusing a DP sync that does not cover this batch's requests"
)
return replace(
target_dp_sync,
num_tokens_across_dp=torch.full_like(
target_dp_sync.num_tokens_across_dp, num_batch_tokens
),
uniform_token_count=num_query_per_req,
eager=False,
), num_batch_tokens