[Model Runner V2][Spec Decode] Skip DP sync before EAGLE/MTP draft prefill (#53694)
This commit is contained in:
@@ -14,7 +14,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
@@ -219,7 +219,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -279,7 +279,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
max_query_len,
|
||||
input_batch.has_prefill,
|
||||
)
|
||||
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
prefill_batch_desc, prefill_batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_tokens_padded,
|
||||
@@ -287,6 +287,12 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=dp_sync,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
prefill_batch_sync.num_tokens_across_dp
|
||||
if prefill_batch_sync is not None
|
||||
else None
|
||||
)
|
||||
|
||||
self._prepare_eplb_forward(num_tokens)
|
||||
@@ -328,7 +334,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
# Each request produces exactly 1 token per draft generation step,
|
||||
# enabling FULL graph replay.
|
||||
decode_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.decode_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_reqs,
|
||||
@@ -337,6 +343,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
decode_batch_sync.num_tokens_across_dp
|
||||
if decode_batch_sync is not None
|
||||
else None
|
||||
)
|
||||
|
||||
self.on_multi_step_decode_begin(num_reqs)
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import cp_local_slot, prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
||||
@@ -335,7 +335,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -377,7 +377,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_tokens,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
@@ -438,7 +440,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.query_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
@@ -450,6 +452,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
)
|
||||
|
||||
# Rebuild the draft attention metadata even when replaying the FULL
|
||||
# graph so that any attention metadata builder state is updated.
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
@@ -97,7 +98,7 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -143,7 +144,9 @@ class ExtractHiddenStatesSpeculator(DraftModelSpeculator):
|
||||
draft_attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
slot_mapping=draft_slot_mappings,
|
||||
is_padding=input_batch.is_padding[:num_tokens],
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
@@ -150,7 +150,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
@@ -191,7 +191,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
uniform_token_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
|
||||
)
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
num_reqs,
|
||||
input_batch.num_tokens_after_padding,
|
||||
@@ -199,6 +199,10 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=dp_sync,
|
||||
)
|
||||
num_tokens_across_dp = (
|
||||
batch_sync.num_tokens_across_dp if batch_sync is not None else None
|
||||
)
|
||||
|
||||
# Rebuild the slot mappings and attention metadata.
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
@@ -60,7 +61,7 @@ class BaseSpeculator(ABC):
|
||||
temperature: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
seeds: torch.Tensor,
|
||||
num_tokens_across_dp: torch.Tensor | None = None,
|
||||
dp_sync: DPSyncState | None = None,
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
|
||||
Reference in New Issue
Block a user