[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
# Supply the resolved Engram settings without constructing a full model.
|
||||
vllm_config.engram_config = EngramConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM", is_moe=True
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
try:
|
||||
init_distributed_environment()
|
||||
|
||||
Reference in New Issue
Block a user