[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)

This commit is contained in:
Giancarlo Delfin
2026-09-03 10:32:33 +00:00
committed by GitHub
parent 0d3ede3e3b
commit facd9a74a1
4 changed files with 48 additions and 7 deletions
+13 -1
View File
@@ -33,6 +33,9 @@ class DPSyncState:
# Whether the ranks agreed to run eager. A dispatch reusing this must run # Whether the ranks agreed to run eager. A dispatch reusing this must run
# eager too; no shape was agreed, so picking a graph per rank would diverge. # eager too; no shape was agreed, so picking a graph per rank would diverge.
eager: bool eager: bool
# Agreed upper bound on any rank's request count. Holds the padded count when
# a FULL descriptor imposed one, else the most any rank scheduled.
num_reqs: int
def sync_cudagraph_and_dp_padding( def sync_cudagraph_and_dp_padding(
@@ -53,17 +56,19 @@ def sync_cudagraph_and_dp_padding(
""" """
assert dp_size > 1, "DP size must be greater than 1" assert dp_size > 1, "DP size must be greater than 1"
group = get_dp_group().cpu_group group = get_dp_group().cpu_group
tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu") tensor = torch.zeros(5, dp_size, dtype=torch.int32, device="cpu")
tensor[0][dp_rank] = num_tokens tensor[0][dp_rank] = num_tokens
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None) tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None) tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
tensor[4][dp_rank] = num_reqs
dist.all_reduce(tensor, group=group) dist.all_reduce(tensor, group=group)
num_tokens_across_dp = tensor[0] num_tokens_across_dp = tensor[0]
cg_mode_across_dp = tensor[1] cg_mode_across_dp = tensor[1]
uniform_token_counts_across_dp = tensor[2] uniform_token_counts_across_dp = tensor[2]
max_query_lens_across_dp = tensor[3] max_query_lens_across_dp = tensor[3]
num_reqs_across_dp = tensor[4]
# If ranks disagree on the uniform token count, or its 0 (means None) set to None # If ranks disagree on the uniform token count, or its 0 (means None) set to None
synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0]) synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0])
@@ -93,6 +98,7 @@ def sync_cudagraph_and_dp_padding(
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
uniform_token_count=synced_uniform_token_count, uniform_token_count=synced_uniform_token_count,
eager=True, eager=True,
num_reqs=int(num_reqs_across_dp.max().item()),
), ),
) )
@@ -126,6 +132,12 @@ def sync_cudagraph_and_dp_padding(
num_tokens_across_dp=num_tokens_across_dp, num_tokens_across_dp=num_tokens_across_dp,
uniform_token_count=synced_uniform_token_count, uniform_token_count=synced_uniform_token_count,
eager=False, eager=False,
num_reqs=(
synced_desc.num_reqs
if synced_desc.cg_mode == CUDAGraphMode.FULL
and synced_desc.num_reqs is not None
else int(num_reqs_across_dp.max().item())
),
) )
@@ -336,16 +336,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
advance_draft_positions=self.advance_draft_positions, advance_draft_positions=self.advance_draft_positions,
) )
decode_batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
if dp_sync is not None
else (None, num_reqs)
)
# Each request produces exactly 1 token per draft generation step, # Each request produces exactly 1 token per draft generation step,
# enabling FULL graph replay. # enabling FULL graph replay.
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp( decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
self.decode_cudagraph_manager, self.decode_cudagraph_manager,
num_reqs, num_reqs,
num_reqs, num_batch_tokens,
uniform_token_count=1, uniform_token_count=1,
dp_size=self.dp_size, dp_size=self.dp_size,
dp_rank=self.dp_rank, dp_rank=self.dp_rank,
need_eager=is_profile, need_eager=is_profile,
dp_sync=decode_batch_sync,
) )
num_tokens_across_dp = ( num_tokens_across_dp = (
decode_batch_sync.num_tokens_across_dp decode_batch_sync.num_tokens_across_dp
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_tokens, num_query_tokens,
attn_metadata=None, attn_metadata=None,
slot_mappings=None, slot_mappings=None,
num_tokens_across_dp=( num_tokens_across_dp=None,
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
cudagraph_runtime_mode=CUDAGraphMode.NONE, cudagraph_runtime_mode=CUDAGraphMode.NONE,
) )
return self.draft_tokens[:num_reqs] return self.draft_tokens[:num_reqs]
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
context_slots, context_slots,
) )
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs # Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
batch_desc, batch_sync = dispatch_cg_and_sync_dp( batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.query_cudagraph_manager, self.query_cudagraph_manager,
num_reqs, num_reqs,
num_query_tokens, num_batch_tokens,
uniform_token_count=self.num_query_per_req, uniform_token_count=self.num_query_per_req,
dp_size=self.dp_size, dp_size=self.dp_size,
dp_rank=self.dp_rank, dp_rank=self.dp_rank,
need_eager=is_profile, need_eager=is_profile,
dp_sync=batch_sync,
) )
num_reqs_padded = batch_desc.num_reqs or num_reqs num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = ( num_tokens_across_dp = (
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Mapping from collections.abc import Mapping
from dataclasses import replace
from typing import Any from typing import Any
import numpy as np import numpy as np
@@ -409,3 +410,22 @@ class DraftModelSpeculator(BaseSpeculator):
# idx_mapping for CG padded requests points to -1, which is ignored # idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits. # during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1) self.idx_mapping[num_reqs:].fill_(-1)
def _build_uniform_batch_dp_sync(
self,
target_dp_sync: DPSyncState,
num_reqs: int,
num_query_per_req: int = 1,
) -> tuple[DPSyncState, int]:
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
assert num_reqs * num_query_per_req <= num_batch_tokens, (
"reusing a DP sync that does not cover this batch's requests"
)
return replace(
target_dp_sync,
num_tokens_across_dp=torch.full_like(
target_dp_sync.num_tokens_across_dp, num_batch_tokens
),
uniform_token_count=num_query_per_req,
eager=False,
), num_batch_tokens