[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)
This commit is contained in:
@@ -336,16 +336,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
advance_draft_positions=self.advance_draft_positions,
|
||||
)
|
||||
|
||||
decode_batch_sync, num_batch_tokens = (
|
||||
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
|
||||
if dp_sync is not None
|
||||
else (None, num_reqs)
|
||||
)
|
||||
# Each request produces exactly 1 token per draft generation step,
|
||||
# enabling FULL graph replay.
|
||||
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.decode_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_reqs,
|
||||
num_batch_tokens,
|
||||
uniform_token_count=1,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=decode_batch_sync,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
decode_batch_sync.num_tokens_across_dp
|
||||
|
||||
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_tokens,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
context_slots,
|
||||
)
|
||||
|
||||
batch_sync, num_batch_tokens = (
|
||||
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
|
||||
if dp_sync is not None
|
||||
else (None, num_query_tokens)
|
||||
)
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.query_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
num_batch_tokens,
|
||||
uniform_token_count=self.num_query_per_req,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=batch_sync,
|
||||
)
|
||||
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -409,3 +410,22 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||
# during sampling to prevent writing stale values to draft logits.
|
||||
self.idx_mapping[num_reqs:].fill_(-1)
|
||||
|
||||
def _build_uniform_batch_dp_sync(
|
||||
self,
|
||||
target_dp_sync: DPSyncState,
|
||||
num_reqs: int,
|
||||
num_query_per_req: int = 1,
|
||||
) -> tuple[DPSyncState, int]:
|
||||
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
|
||||
assert num_reqs * num_query_per_req <= num_batch_tokens, (
|
||||
"reusing a DP sync that does not cover this batch's requests"
|
||||
)
|
||||
return replace(
|
||||
target_dp_sync,
|
||||
num_tokens_across_dp=torch.full_like(
|
||||
target_dp_sync.num_tokens_across_dp, num_batch_tokens
|
||||
),
|
||||
uniform_token_count=num_query_per_req,
|
||||
eager=False,
|
||||
), num_batch_tokens
|
||||
|
||||
Reference in New Issue
Block a user