[Bugfix] Fix dp mtp hang (#40589)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: sherryC41 <sherry.c.c41@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
SherryC41
2026-07-06 21:08:17 +00:00
committed by GitHub
co-authored by sherryC41 Matthew Bonanni
parent d891b9bd51
commit 567a78432d
+45 -18
View File
@@ -4527,17 +4527,23 @@ class GPUModelRunner(
self._copy_draft_token_ids_to_cpu(scheduler_output)
spec_config = self.speculative_config
propose_drafts_after_bookkeeping = False
draft_after_bookkeeping = False
if spec_config is not None:
# Decide whether to run the drafter or zero out draft tokens.
input_fits_in_drafter = self._input_fits_in_drafter(
spec_decode_common_attn_metadata
)
use_gpu_toks = (
# Whether the drafter runs a GPU model forward (and thus carries
# TP/EP/DP collectives), independent of padded-batch timing.
drafter_runs_model_forward = (
spec_config.use_eagle()
or spec_config.uses_draft_model()
or spec_config.uses_extract_hidden_states()
) and not spec_config.disable_padded_drafter_batch
)
use_gpu_toks = (
drafter_runs_model_forward
and not spec_config.disable_padded_drafter_batch
)
if use_gpu_toks:
# EAGLE/DraftModel speculative decoding can use the GPU sampled tokens
# as inputs, and does not need to wait for bookkeeping to finish.
@@ -4552,19 +4558,23 @@ class GPUModelRunner(
sampled_token_ids = sampler_output.sampled_token_ids
if input_fits_in_drafter:
propose_draft_token_ids(sampled_token_ids)
elif self.valid_sampled_token_count_event is not None:
assert spec_decode_common_attn_metadata is not None
next_token_ids, valid_sampled_tokens_count = (
self.drafter.prepare_next_token_ids_padded(
sampled_token_ids,
self.requests,
self.input_batch,
self.discard_request_mask.gpu,
else:
if self.valid_sampled_token_count_event is not None:
assert spec_decode_common_attn_metadata is not None
next_token_ids, valid_sampled_tokens_count = (
self.drafter.prepare_next_token_ids_padded(
sampled_token_ids,
self.requests,
self.input_batch,
self.discard_request_mask.gpu,
)
)
)
self._copy_valid_sampled_token_count(
next_token_ids, valid_sampled_tokens_count
)
self._copy_valid_sampled_token_count(
next_token_ids, valid_sampled_tokens_count
)
if self.parallel_config.data_parallel_size > 1:
# Prevent hang when DP ranks disagree on input_fits_in_drafter
self.drafter.dummy_run(num_tokens=1)
elif (
spec_config.use_ngram_gpu()
and not spec_config.disable_padded_drafter_batch
@@ -4588,7 +4598,9 @@ class GPUModelRunner(
next_token_ids, valid_sampled_tokens_count
)
else:
propose_drafts_after_bookkeeping = input_fits_in_drafter
# These drafters consume CPU sampled tokens, so they run
# after bookkeeping.
draft_after_bookkeeping = True
if not input_fits_in_drafter:
# Zero out draft tokens so the scheduler doesn't schedule
@@ -4620,10 +4632,25 @@ class GPUModelRunner(
scheduler_output.total_num_scheduled_tokens,
)
if propose_drafts_after_bookkeeping:
if draft_after_bookkeeping:
# ngram and other speculative decoding methods use the sampled
# tokens on the CPU, so they are run after bookkeeping.
propose_draft_token_ids(valid_sampled_token_ids)
if input_fits_in_drafter:
propose_draft_token_ids(valid_sampled_token_ids)
elif (
drafter_runs_model_forward
and self.parallel_config.data_parallel_size > 1
):
# Prevent hang when DP ranks disagree on input_fits_in_drafter
assert isinstance(
self.drafter,
EagleProposer
| DFlashProposer
| DraftModelProposer
| ExtractHiddenStatesProposer
| Gemma4Proposer,
)
self.drafter.dummy_run(num_tokens=1)
# Finalize KV connector (wait_for_save + clear metadata) after
# draft model runs. Deferred from target model forward to allow