[Bugfix] Don't drop the rest of the allocator config when toggling expandable segments (#57982)
Signed-off-by: Oxana Korzh <okorzh@amd.com>
This commit is contained in:
@@ -10,7 +10,6 @@
|
||||
# the only successful approach is to call cuda driver API in C.
|
||||
import atexit
|
||||
import gc
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
@@ -18,6 +17,13 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.device_allocator.alloc_conf import (
|
||||
EXPANDABLE_SEGMENTS,
|
||||
conf_flag_enabled,
|
||||
current_alloc_conf,
|
||||
set_alloc_conf,
|
||||
with_conf_flag,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import find_loaded_library
|
||||
@@ -371,13 +377,17 @@ class CuMemAllocator:
|
||||
|
||||
# Expandable segments are incompatible with the memory pool used for
|
||||
# sleep mode (see https://github.com/pytorch/pytorch/issues/147851).
|
||||
# If the user has enabled expandable segments via
|
||||
# PYTORCH_CUDA_ALLOC_CONF, temporarily disable them for the duration
|
||||
# of the memory pool context and restore on exit.
|
||||
conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
|
||||
expandable_was_enabled = "expandable_segments:True" in conf
|
||||
# If the user has enabled expandable segments, temporarily disable
|
||||
# them for the duration of the memory pool context and restore on
|
||||
# exit. The whole config string is rewritten, not just this one
|
||||
# field: torch resets every option that is absent from the string it
|
||||
# is handed, so a bare "expandable_segments:False" would also drop
|
||||
# max_split_size_mb, garbage_collection_threshold and
|
||||
# roundup_power2_divisions for the rest of the process.
|
||||
prev_conf = current_alloc_conf()
|
||||
expandable_was_enabled = conf_flag_enabled(prev_conf, EXPANDABLE_SEGMENTS)
|
||||
if expandable_was_enabled:
|
||||
torch.cuda.memory._set_allocator_settings("expandable_segments:False")
|
||||
set_alloc_conf(with_conf_flag(prev_conf, EXPANDABLE_SEGMENTS, False))
|
||||
|
||||
old_tag = self.current_tag
|
||||
self.current_tag = tag
|
||||
@@ -411,7 +421,7 @@ class CuMemAllocator:
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
if expandable_was_enabled:
|
||||
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
|
||||
set_alloc_conf(prev_conf)
|
||||
|
||||
def get_current_usage(self) -> int:
|
||||
"""Get the total number of bytes allocated in the memory pool."""
|
||||
|
||||
Reference in New Issue
Block a user