[Core] Whisper support torch.compile (#30385)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-01-19 10:02:31 +00:00
committed by GitHub
parent c0a350ca73
commit 74c583bc50
5 changed files with 27 additions and 1 deletions
+8
View File
@@ -3268,6 +3268,13 @@ class GPUModelRunner(
# Mark KV scales as calculated after the first forward pass
self.calculate_kv_scales = False
# Encoder-decoder models can only compile the pure decode steps where no
# encoder inputs are present. Use eager for the first pass.
num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs)
has_encoder_input = (
self.model_config.is_encoder_decoder and num_encoder_reqs > 0
)
# Run the model.
# Use persistent buffers for CUDA graphs.
with (
@@ -3279,6 +3286,7 @@ class GPUModelRunner(
cudagraph_runtime_mode=cudagraph_mode,
batch_descriptor=batch_desc,
ubatch_slices=ubatch_slices_padded,
skip_compiled=has_encoder_input,
),
record_function_or_nullcontext("gpu_model_runner: forward"),
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,