[Core] Release NCCL communicator memory in sleep mode (#51485)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-01 14:31:16 +08:00
committed by GitHub
co-authored by aoshen02 Claude Fable 5
parent ec32f669bb
commit 4ac452ad98
12 changed files with 182 additions and 6 deletions
+6
View File
@@ -710,6 +710,7 @@ class EngineArgs:
generation_config: str = ModelConfig.generation_config
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend
override_generation_config: dict[str, Any] = get_field(
ModelConfig, "override_generation_config"
)
@@ -932,6 +933,10 @@ class EngineArgs:
model_group.add_argument(
"--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
)
model_group.add_argument(
"--enable-nccl-comm-suspend",
**model_kwargs["enable_nccl_comm_suspend"],
)
model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
model_group.add_argument(
"--logits-processors", **model_kwargs["logits_processors"]
@@ -1823,6 +1828,7 @@ class EngineArgs:
override_generation_config=self.override_generation_config,
enable_sleep_mode=self.enable_sleep_mode,
enable_cumem_allocator=self.enable_cumem_allocator,
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
model_impl=self.model_impl,
logits_processors=self.logits_processors,
video_pruning_rate=self.video_pruning_rate,