[Bugfix] Don't drop the rest of the allocator config when toggling expandable segments (#57982)

Signed-off-by: Oxana Korzh <okorzh@amd.com>
This commit is contained in:
Oxana Korzh
2026-09-25 17:34:19 -05:00
committed by GitHub
parent 4ccfe12398
commit e55d076f89
4 changed files with 270 additions and 8 deletions
+18 -8
View File
@@ -10,7 +10,6 @@
# the only successful approach is to call cuda driver API in C.
import atexit
import gc
import os
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
@@ -18,6 +17,13 @@ from typing import Any
import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.device_allocator.alloc_conf import (
EXPANDABLE_SEGMENTS,
conf_flag_enabled,
current_alloc_conf,
set_alloc_conf,
with_conf_flag,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.system_utils import find_loaded_library
@@ -371,13 +377,17 @@ class CuMemAllocator:
# Expandable segments are incompatible with the memory pool used for
# sleep mode (see https://github.com/pytorch/pytorch/issues/147851).
# If the user has enabled expandable segments via
# PYTORCH_CUDA_ALLOC_CONF, temporarily disable them for the duration
# of the memory pool context and restore on exit.
conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
expandable_was_enabled = "expandable_segments:True" in conf
# If the user has enabled expandable segments, temporarily disable
# them for the duration of the memory pool context and restore on
# exit. The whole config string is rewritten, not just this one
# field: torch resets every option that is absent from the string it
# is handed, so a bare "expandable_segments:False" would also drop
# max_split_size_mb, garbage_collection_threshold and
# roundup_power2_divisions for the rest of the process.
prev_conf = current_alloc_conf()
expandable_was_enabled = conf_flag_enabled(prev_conf, EXPANDABLE_SEGMENTS)
if expandable_was_enabled:
torch.cuda.memory._set_allocator_settings("expandable_segments:False")
set_alloc_conf(with_conf_flag(prev_conf, EXPANDABLE_SEGMENTS, False))
old_tag = self.current_tag
self.current_tag = tag
@@ -411,7 +421,7 @@ class CuMemAllocator:
finally:
self.current_tag = old_tag
if expandable_was_enabled:
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
set_alloc_conf(prev_conf)
def get_current_usage(self) -> int:
"""Get the total number of bytes allocated in the memory pool."""