[BUGFIX] Fix _dummy_run missing prepare_inputs_event synchronization (#34866)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
@@ -4771,34 +4771,39 @@ class GPUModelRunner(
|
|||||||
ubatch_slices=ubatch_slices_padded,
|
ubatch_slices=ubatch_slices_padded,
|
||||||
)
|
)
|
||||||
|
|
||||||
# If force_attention is True, we always capture attention. Otherwise,
|
# _dummy_run shares pinned CPU buffers (seq_lens, query_start_loc,
|
||||||
# it only happens for cudagraph_runtime_mode=FULL.
|
# etc.) with execute_model. It must participate in the same event
|
||||||
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
# protocol so that back-to-back dummy/real steps don't overwrite
|
||||||
if create_mixed_batch:
|
# pinned memory while a prior non_blocking H2D DMA is still reading.
|
||||||
# In the mixed batch mode (used for FI warmup), we use
|
with self.synchronize_input_prep():
|
||||||
# shorter sequence lengths to run faster.
|
# If force_attention is True, we always capture attention.
|
||||||
# TODO(luka) better system for describing dummy batches
|
# Otherwise, it only happens for cudagraph_runtime_mode=FULL.
|
||||||
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1]
|
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||||
else:
|
if create_mixed_batch:
|
||||||
seq_lens = max_query_len # type: ignore[assignment]
|
# In the mixed batch mode (used for FI warmup), we use
|
||||||
self.seq_lens.np[:num_reqs] = seq_lens
|
# shorter sequence lengths to run faster.
|
||||||
self.seq_lens.np[num_reqs:] = 0
|
# TODO(luka) better system for describing dummy batches
|
||||||
self.seq_lens.copy_to_gpu()
|
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1]
|
||||||
|
else:
|
||||||
|
seq_lens = max_query_len # type: ignore[assignment]
|
||||||
|
self.seq_lens.np[:num_reqs] = seq_lens
|
||||||
|
self.seq_lens.np[num_reqs:] = 0
|
||||||
|
self.seq_lens.copy_to_gpu()
|
||||||
|
|
||||||
cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens)
|
cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens)
|
||||||
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
|
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
|
||||||
self.query_start_loc.copy_to_gpu()
|
self.query_start_loc.copy_to_gpu()
|
||||||
|
|
||||||
pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL
|
pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||||
attn_metadata, _ = self._build_attention_metadata(
|
attn_metadata, _ = self._build_attention_metadata(
|
||||||
num_tokens=num_tokens_unpadded,
|
num_tokens=num_tokens_unpadded,
|
||||||
num_tokens_padded=num_tokens_padded if pad_attn else None,
|
num_tokens_padded=num_tokens_padded if pad_attn else None,
|
||||||
num_reqs=num_reqs_padded,
|
num_reqs=num_reqs_padded,
|
||||||
max_query_len=max_query_len,
|
max_query_len=max_query_len,
|
||||||
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
|
ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices),
|
||||||
for_cudagraph_capture=is_graph_capturing,
|
for_cudagraph_capture=is_graph_capturing,
|
||||||
slot_mappings=slot_mappings_by_group,
|
slot_mappings=slot_mappings_by_group,
|
||||||
)
|
)
|
||||||
|
|
||||||
with self.maybe_dummy_run_with_lora(
|
with self.maybe_dummy_run_with_lora(
|
||||||
self.lora_config,
|
self.lora_config,
|
||||||
|
|||||||
Reference in New Issue
Block a user