[Model Runner V2][Spec Decode] Skip DP sync before EAGLE/MTP draft prefill (#53694)
This commit is contained in:
@@ -97,7 +97,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
profile_cudagraph_memory as _profile_cudagraph_memory,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
||||
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
@@ -752,6 +752,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
self.execute_model_state = None
|
||||
|
||||
self.step_timing.forward_end()
|
||||
@@ -796,6 +797,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
dp_sync=dp_sync,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
@@ -1544,7 +1546,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# cross-attention cache with dynamic encoder outputs.
|
||||
skip_compiled = True
|
||||
|
||||
batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
batch_desc, dp_sync = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
num_reqs,
|
||||
num_toks,
|
||||
@@ -1731,7 +1733,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
num_tokens=input_batch.num_tokens_after_padding,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
batch_descriptor=batch_descriptor,
|
||||
slot_mapping=slot_mappings_by_layer,
|
||||
skip_compiled=skip_compiled,
|
||||
@@ -1777,6 +1781,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer=slot_mappings_by_layer,
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
dp_sync=dp_sync,
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
routed_experts=routed_experts,
|
||||
@@ -1801,6 +1806,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
dp_sync = self.execute_model_state.dp_sync
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
routed_experts = self.execute_model_state.routed_experts
|
||||
@@ -1922,6 +1928,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.next_prefill_tokens,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
dp_sync=dp_sync,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
@@ -2066,6 +2073,7 @@ class ExecuteModelState(NamedTuple):
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] | None
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
dp_sync: DPSyncState | None
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
routed_experts: RoutedExpertsTensors | None
|
||||
|
||||
Reference in New Issue
Block a user