[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -842,10 +842,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return 0
|
||||
|
||||
assert self.cudagraph_manager is not None
|
||||
if not self.cudagraph_manager.needs_capture():
|
||||
capture_encoder = (
|
||||
self.model_state.supports_mm_inputs
|
||||
and self.model_state.encoder_runner.has_cudagraph()
|
||||
)
|
||||
capture_decoder = self.cudagraph_manager.needs_capture()
|
||||
if not capture_encoder and not capture_decoder:
|
||||
logger.warning(
|
||||
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
|
||||
"ensure `cudagraph_mode` was not manually set to `NONE`"
|
||||
"Skipping encoder and decoder CUDA graph capture. To enable "
|
||||
"encoder capture, ensure `cudagraph_mm_encoder` is enabled; "
|
||||
"to enable decoder capture, ensure `cudagraph_mode` is not `NONE`."
|
||||
)
|
||||
return 0
|
||||
|
||||
@@ -857,27 +863,31 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
|
||||
if capture_decoder:
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
@@ -1885,6 +1895,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.attn_groups.clear()
|
||||
if hasattr(self, "kv_cache_config"):
|
||||
del self.kv_cache_config
|
||||
if hasattr(self, "model_state") and self.model_state.supports_mm_inputs:
|
||||
self.model_state.encoder_runner.clear()
|
||||
free_before_shutdown(self.vllm_config)
|
||||
if hasattr(self, "model_state"):
|
||||
del self.model_state
|
||||
|
||||
Reference in New Issue
Block a user