[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Isotr0py
2026-08-14 09:33:21 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b8165e5e58
commit 66728feb1f
4 changed files with 101 additions and 26 deletions
+36 -24
View File
@@ -842,10 +842,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return 0
assert self.cudagraph_manager is not None
if not self.cudagraph_manager.needs_capture():
capture_encoder = (
self.model_state.supports_mm_inputs
and self.model_state.encoder_runner.has_cudagraph()
)
capture_decoder = self.cudagraph_manager.needs_capture()
if not capture_encoder and not capture_decoder:
logger.warning(
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
"ensure `cudagraph_mode` was not manually set to `NONE`"
"Skipping encoder and decoder CUDA graph capture. To enable "
"encoder capture, ensure `cudagraph_mm_encoder` is enabled; "
"to enable decoder capture, ensure `cudagraph_mode` is not `NONE`."
)
return 0
@@ -857,27 +863,31 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
if capture_encoder:
self.model_state.encoder_runner.capture()
if capture_decoder:
self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
end_time = time.perf_counter()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -1885,6 +1895,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.attn_groups.clear()
if hasattr(self, "kv_cache_config"):
del self.kv_cache_config
if hasattr(self, "model_state") and self.model_state.supports_mm_inputs:
self.model_state.encoder_runner.clear()
free_before_shutdown(self.vllm_config)
if hasattr(self, "model_state"):
del self.model_state