[Model Runner V2][Spec Decode] Skip DP sync before EAGLE/MTP draft prefill (#53694)
This commit is contained in:
@@ -19,7 +19,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
||||
@@ -335,7 +335,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -377,7 +377,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_tokens,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
@@ -438,7 +440,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.query_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
@@ -450,6 +452,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
)
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
|
||||
Reference in New Issue
Block a user