[Model Runner V2] Support randomized dummy inputs (#58411)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Robert Shaw
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
parent
5addde2a0a
commit
e5e38ba9b7
@@ -5735,6 +5735,9 @@ class GPUModelRunner(
|
||||
- during profile_run
|
||||
- during DP rank dummy run
|
||||
"""
|
||||
# The worker also resolves this flag and passes randomize_inputs, but
|
||||
# V1 has internal dummy runs (e.g. CUDA graph capture) that don't go
|
||||
# through the worker, so keep the check here too.
|
||||
dp_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
randomize_inputs = randomize_inputs or (
|
||||
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
|
||||
@@ -6420,7 +6423,7 @@ class GPUModelRunner(
|
||||
max_task = max(output_size.items(), key=lambda x: x[1])[0]
|
||||
return self._dummy_pooler_run_task(hidden_states, max_task)
|
||||
|
||||
def profile_run(self) -> None:
|
||||
def profile_run(self, randomize_inputs: bool = False) -> None:
|
||||
# Profile with multimodal encoder & encoder cache.
|
||||
if self.supports_mm_inputs:
|
||||
mm_config = self.model_config.multimodal_config
|
||||
@@ -6481,7 +6484,7 @@ class GPUModelRunner(
|
||||
|
||||
# Add `is_profile` here to pre-allocate communication buffers
|
||||
hidden_states, last_hidden_states = self._dummy_run(
|
||||
self.max_num_tokens, is_profile=True
|
||||
self.max_num_tokens, is_profile=True, randomize_inputs=randomize_inputs
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
if self.is_pooling_model:
|
||||
|
||||
Reference in New Issue
Block a user