diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 2cf7d2db54..24889f4021 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -4527,17 +4527,23 @@ class GPUModelRunner( self._copy_draft_token_ids_to_cpu(scheduler_output) spec_config = self.speculative_config - propose_drafts_after_bookkeeping = False + draft_after_bookkeeping = False if spec_config is not None: # Decide whether to run the drafter or zero out draft tokens. input_fits_in_drafter = self._input_fits_in_drafter( spec_decode_common_attn_metadata ) - use_gpu_toks = ( + # Whether the drafter runs a GPU model forward (and thus carries + # TP/EP/DP collectives), independent of padded-batch timing. + drafter_runs_model_forward = ( spec_config.use_eagle() or spec_config.uses_draft_model() or spec_config.uses_extract_hidden_states() - ) and not spec_config.disable_padded_drafter_batch + ) + use_gpu_toks = ( + drafter_runs_model_forward + and not spec_config.disable_padded_drafter_batch + ) if use_gpu_toks: # EAGLE/DraftModel speculative decoding can use the GPU sampled tokens # as inputs, and does not need to wait for bookkeeping to finish. @@ -4552,19 +4558,23 @@ class GPUModelRunner( sampled_token_ids = sampler_output.sampled_token_ids if input_fits_in_drafter: propose_draft_token_ids(sampled_token_ids) - elif self.valid_sampled_token_count_event is not None: - assert spec_decode_common_attn_metadata is not None - next_token_ids, valid_sampled_tokens_count = ( - self.drafter.prepare_next_token_ids_padded( - sampled_token_ids, - self.requests, - self.input_batch, - self.discard_request_mask.gpu, + else: + if self.valid_sampled_token_count_event is not None: + assert spec_decode_common_attn_metadata is not None + next_token_ids, valid_sampled_tokens_count = ( + self.drafter.prepare_next_token_ids_padded( + sampled_token_ids, + self.requests, + self.input_batch, + self.discard_request_mask.gpu, + ) ) - ) - self._copy_valid_sampled_token_count( - next_token_ids, valid_sampled_tokens_count - ) + self._copy_valid_sampled_token_count( + next_token_ids, valid_sampled_tokens_count + ) + if self.parallel_config.data_parallel_size > 1: + # Prevent hang when DP ranks disagree on input_fits_in_drafter + self.drafter.dummy_run(num_tokens=1) elif ( spec_config.use_ngram_gpu() and not spec_config.disable_padded_drafter_batch @@ -4588,7 +4598,9 @@ class GPUModelRunner( next_token_ids, valid_sampled_tokens_count ) else: - propose_drafts_after_bookkeeping = input_fits_in_drafter + # These drafters consume CPU sampled tokens, so they run + # after bookkeeping. + draft_after_bookkeeping = True if not input_fits_in_drafter: # Zero out draft tokens so the scheduler doesn't schedule @@ -4620,10 +4632,25 @@ class GPUModelRunner( scheduler_output.total_num_scheduled_tokens, ) - if propose_drafts_after_bookkeeping: + if draft_after_bookkeeping: # ngram and other speculative decoding methods use the sampled # tokens on the CPU, so they are run after bookkeeping. - propose_draft_token_ids(valid_sampled_token_ids) + if input_fits_in_drafter: + propose_draft_token_ids(valid_sampled_token_ids) + elif ( + drafter_runs_model_forward + and self.parallel_config.data_parallel_size > 1 + ): + # Prevent hang when DP ranks disagree on input_fits_in_drafter + assert isinstance( + self.drafter, + EagleProposer + | DFlashProposer + | DraftModelProposer + | ExtractHiddenStatesProposer + | Gemma4Proposer, + ) + self.drafter.dummy_run(num_tokens=1) # Finalize KV connector (wait_for_save + clear metadata) after # draft model runs. Deferred from target model forward to allow