[Model Runner V2][Spec Decode] Skip DP sync before EAGLE/MTP draft prefill (#53694)

This commit is contained in:
Giancarlo Delfin
2026-08-26 20:59:35 -07:00
committed by GitHub
parent ca90b9e7d4
commit 5acc1c4e4b
8 changed files with 156 additions and 36 deletions
+11 -3
View File
@@ -97,7 +97,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
from vllm.v1.worker.gpu.cudagraph_utils import (
profile_cudagraph_memory as _profile_cudagraph_memory,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
from vllm.v1.worker.gpu.input_batch import (
@@ -752,6 +752,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
self.execute_model_state = None
self.step_timing.forward_end()
@@ -796,6 +797,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
next_prefill_tokens=self.req_states.next_prefill_tokens,
temperature=self.sampler.sampling_states.temperature.gpu,
seeds=self.sampler.sampling_states.seeds.gpu,
dp_sync=dp_sync,
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
mm_inputs=mm_inputs,
@@ -1544,7 +1546,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# cross-attention cache with dynamic encoder outputs.
skip_compiled = True
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
self.cudagraph_manager,
num_reqs,
num_toks,
@@ -1731,7 +1733,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.vllm_config,
num_tokens=input_batch.num_tokens_after_padding,
cudagraph_runtime_mode=batch_desc.cg_mode,
num_tokens_across_dp=num_tokens_across_dp,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
batch_descriptor=batch_descriptor,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
@@ -1777,6 +1781,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer=slot_mappings_by_layer,
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
dp_sync=dp_sync,
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
routed_experts=routed_experts,
@@ -1801,6 +1806,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
dp_sync = self.execute_model_state.dp_sync
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
routed_experts = self.execute_model_state.routed_experts
@@ -1922,6 +1928,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.next_prefill_tokens,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
dp_sync=dp_sync,
mm_inputs=mm_inputs,
)
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
@@ -2066,6 +2073,7 @@ class ExecuteModelState(NamedTuple):
slot_mappings_by_layer: dict[str, torch.Tensor] | None
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
dp_sync: DPSyncState | None
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
routed_experts: RoutedExpertsTensors | None