[Model Runner V2] Support randomized dummy inputs (#58411)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Robert Shaw
2026-10-01 00:52:26 -07:00
committed by GitHub
co-authored by Robert Shaw Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill
parent 5addde2a0a
commit e5e38ba9b7
4 changed files with 43 additions and 9 deletions
+5 -2
View File
@@ -5735,6 +5735,9 @@ class GPUModelRunner(
- during profile_run
- during DP rank dummy run
"""
# The worker also resolves this flag and passes randomize_inputs, but
# V1 has internal dummy runs (e.g. CUDA graph capture) that don't go
# through the worker, so keep the check here too.
dp_size = self.vllm_config.parallel_config.data_parallel_size
randomize_inputs = randomize_inputs or (
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
@@ -6420,7 +6423,7 @@ class GPUModelRunner(
max_task = max(output_size.items(), key=lambda x: x[1])[0]
return self._dummy_pooler_run_task(hidden_states, max_task)
def profile_run(self) -> None:
def profile_run(self, randomize_inputs: bool = False) -> None:
# Profile with multimodal encoder & encoder cache.
if self.supports_mm_inputs:
mm_config = self.model_config.multimodal_config
@@ -6481,7 +6484,7 @@ class GPUModelRunner(
# Add `is_profile` here to pre-allocate communication buffers
hidden_states, last_hidden_states = self._dummy_run(
self.max_num_tokens, is_profile=True
self.max_num_tokens, is_profile=True, randomize_inputs=randomize_inputs
)
if get_pp_group().is_last_rank:
if self.is_pooling_model: