[Feature] Support manually enabling the cumem allocator (#33648)

Signed-off-by: Kebe <mail@kebe7jun.com>
This commit is contained in:
Kebe
2026-05-20 08:58:30 -04:00
committed by GitHub
parent 87e31455b0
commit 19cf334207
6 changed files with 60 additions and 17 deletions
+5
View File
@@ -655,6 +655,7 @@ class EngineArgs:
generation_config: str = ModelConfig.generation_config
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
override_generation_config: dict[str, Any] = get_field(
ModelConfig, "override_generation_config"
)
@@ -839,6 +840,9 @@ class EngineArgs:
model_group.add_argument(
"--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
)
model_group.add_argument(
"--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
)
model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
model_group.add_argument(
"--override-attention-dtype", **model_kwargs["override_attention_dtype"]
@@ -1590,6 +1594,7 @@ class EngineArgs:
generation_config=self.generation_config,
override_generation_config=self.override_generation_config,
enable_sleep_mode=self.enable_sleep_mode,
enable_cumem_allocator=self.enable_cumem_allocator,
model_impl=self.model_impl,
override_attention_dtype=self.override_attention_dtype,
logits_processors=self.logits_processors,