[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -710,6 +710,7 @@ class EngineArgs:
|
||||
generation_config: str = ModelConfig.generation_config
|
||||
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
|
||||
enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
|
||||
enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend
|
||||
override_generation_config: dict[str, Any] = get_field(
|
||||
ModelConfig, "override_generation_config"
|
||||
)
|
||||
@@ -932,6 +933,10 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--enable-nccl-comm-suspend",
|
||||
**model_kwargs["enable_nccl_comm_suspend"],
|
||||
)
|
||||
model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
|
||||
model_group.add_argument(
|
||||
"--logits-processors", **model_kwargs["logits_processors"]
|
||||
@@ -1823,6 +1828,7 @@ class EngineArgs:
|
||||
override_generation_config=self.override_generation_config,
|
||||
enable_sleep_mode=self.enable_sleep_mode,
|
||||
enable_cumem_allocator=self.enable_cumem_allocator,
|
||||
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
|
||||
model_impl=self.model_impl,
|
||||
logits_processors=self.logits_processors,
|
||||
video_pruning_rate=self.video_pruning_rate,
|
||||
|
||||
Reference in New Issue
Block a user