[Core] Whisper support torch.compile (#30385)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -3268,6 +3268,13 @@ class GPUModelRunner(
|
||||
# Mark KV scales as calculated after the first forward pass
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
# Encoder-decoder models can only compile the pure decode steps where no
|
||||
# encoder inputs are present. Use eager for the first pass.
|
||||
num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs)
|
||||
has_encoder_input = (
|
||||
self.model_config.is_encoder_decoder and num_encoder_reqs > 0
|
||||
)
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
with (
|
||||
@@ -3279,6 +3286,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=cudagraph_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
skip_compiled=has_encoder_input,
|
||||
),
|
||||
record_function_or_nullcontext("gpu_model_runner: forward"),
|
||||
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,
|
||||
|
||||
Reference in New Issue
Block a user