[Bugfix] Fix dp mtp hang (#40589)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: sherryC41 <sherry.c.c41@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
sherryC41
Matthew Bonanni
parent
d891b9bd51
commit
567a78432d
@@ -4527,17 +4527,23 @@ class GPUModelRunner(
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output)
|
||||
|
||||
spec_config = self.speculative_config
|
||||
propose_drafts_after_bookkeeping = False
|
||||
draft_after_bookkeeping = False
|
||||
if spec_config is not None:
|
||||
# Decide whether to run the drafter or zero out draft tokens.
|
||||
input_fits_in_drafter = self._input_fits_in_drafter(
|
||||
spec_decode_common_attn_metadata
|
||||
)
|
||||
use_gpu_toks = (
|
||||
# Whether the drafter runs a GPU model forward (and thus carries
|
||||
# TP/EP/DP collectives), independent of padded-batch timing.
|
||||
drafter_runs_model_forward = (
|
||||
spec_config.use_eagle()
|
||||
or spec_config.uses_draft_model()
|
||||
or spec_config.uses_extract_hidden_states()
|
||||
) and not spec_config.disable_padded_drafter_batch
|
||||
)
|
||||
use_gpu_toks = (
|
||||
drafter_runs_model_forward
|
||||
and not spec_config.disable_padded_drafter_batch
|
||||
)
|
||||
if use_gpu_toks:
|
||||
# EAGLE/DraftModel speculative decoding can use the GPU sampled tokens
|
||||
# as inputs, and does not need to wait for bookkeeping to finish.
|
||||
@@ -4552,19 +4558,23 @@ class GPUModelRunner(
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
if input_fits_in_drafter:
|
||||
propose_draft_token_ids(sampled_token_ids)
|
||||
elif self.valid_sampled_token_count_event is not None:
|
||||
assert spec_decode_common_attn_metadata is not None
|
||||
next_token_ids, valid_sampled_tokens_count = (
|
||||
self.drafter.prepare_next_token_ids_padded(
|
||||
sampled_token_ids,
|
||||
self.requests,
|
||||
self.input_batch,
|
||||
self.discard_request_mask.gpu,
|
||||
else:
|
||||
if self.valid_sampled_token_count_event is not None:
|
||||
assert spec_decode_common_attn_metadata is not None
|
||||
next_token_ids, valid_sampled_tokens_count = (
|
||||
self.drafter.prepare_next_token_ids_padded(
|
||||
sampled_token_ids,
|
||||
self.requests,
|
||||
self.input_batch,
|
||||
self.discard_request_mask.gpu,
|
||||
)
|
||||
)
|
||||
)
|
||||
self._copy_valid_sampled_token_count(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
self._copy_valid_sampled_token_count(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
if self.parallel_config.data_parallel_size > 1:
|
||||
# Prevent hang when DP ranks disagree on input_fits_in_drafter
|
||||
self.drafter.dummy_run(num_tokens=1)
|
||||
elif (
|
||||
spec_config.use_ngram_gpu()
|
||||
and not spec_config.disable_padded_drafter_batch
|
||||
@@ -4588,7 +4598,9 @@ class GPUModelRunner(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
else:
|
||||
propose_drafts_after_bookkeeping = input_fits_in_drafter
|
||||
# These drafters consume CPU sampled tokens, so they run
|
||||
# after bookkeeping.
|
||||
draft_after_bookkeeping = True
|
||||
|
||||
if not input_fits_in_drafter:
|
||||
# Zero out draft tokens so the scheduler doesn't schedule
|
||||
@@ -4620,10 +4632,25 @@ class GPUModelRunner(
|
||||
scheduler_output.total_num_scheduled_tokens,
|
||||
)
|
||||
|
||||
if propose_drafts_after_bookkeeping:
|
||||
if draft_after_bookkeeping:
|
||||
# ngram and other speculative decoding methods use the sampled
|
||||
# tokens on the CPU, so they are run after bookkeeping.
|
||||
propose_draft_token_ids(valid_sampled_token_ids)
|
||||
if input_fits_in_drafter:
|
||||
propose_draft_token_ids(valid_sampled_token_ids)
|
||||
elif (
|
||||
drafter_runs_model_forward
|
||||
and self.parallel_config.data_parallel_size > 1
|
||||
):
|
||||
# Prevent hang when DP ranks disagree on input_fits_in_drafter
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer
|
||||
| DFlashProposer
|
||||
| DraftModelProposer
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer,
|
||||
)
|
||||
self.drafter.dummy_run(num_tokens=1)
|
||||
|
||||
# Finalize KV connector (wait_for_save + clear metadata) after
|
||||
# draft model runs. Deferred from target model forward to allow
|
||||
|
||||
Reference in New Issue
Block a user