[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
+6
View File
@@ -770,6 +770,7 @@ class EngineArgs:
generation_config: str = ModelConfig.generation_config
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
sleep_mode_offload_cudagraph: bool = ModelConfig.sleep_mode_offload_cudagraph
sleep_preserve_parameter_names: list[str] = get_field(
ModelConfig, "sleep_preserve_parameter_names"
)
@@ -1007,6 +1008,10 @@ class EngineArgs:
model_group.add_argument(
"--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
)
model_group.add_argument(
"--sleep-mode-offload-cudagraph",
**model_kwargs["sleep_mode_offload_cudagraph"],
)
model_group.add_argument(
"--sleep-preserve-parameter-names",
**model_kwargs["sleep_preserve_parameter_names"],
@@ -1984,6 +1989,7 @@ class EngineArgs:
generation_config=self.generation_config,
override_generation_config=self.override_generation_config,
enable_sleep_mode=self.enable_sleep_mode,
sleep_mode_offload_cudagraph=self.sleep_mode_offload_cudagraph,
sleep_preserve_parameter_names=self.sleep_preserve_parameter_names,
enable_cumem_allocator=self.enable_cumem_allocator,
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,