[Feature] Release WorkspaceManager scratch on sleep (#59156)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 02:14:37 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e5d33a36e7
commit 866f02db8a
6 changed files with 140 additions and 23 deletions
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import pytest import pytest
import torch import torch
@@ -233,6 +235,98 @@ def test_discard_tags():
assert torch.allclose(weights, torch.ones_like(weights)) assert torch.allclose(weights, torch.ones_like(weights))
@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_selective_wake_restores_internal_tags(tag):
"""A selective wake defers only weights/kv_cache; other tags always wake."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool(tag):
internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
allocator.sleep(offload_tags=())
assert mapped_usage(allocator) == 0
allocator.wake_up(tags=["weights"])
assert mapped_usage(allocator) == internal.nbytes
allocator.wake_up(tags=["kv_cache"])
assert mapped_usage(allocator) == internal.nbytes + kv.nbytes
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_workspace_scratch_discarded_on_sleep():
"""Workspace scratch lives in the sleep-mode pool: sleep discards it, any
(selective) wake remaps it at the same address so captured graphs replay,
and a buffer outgrown by a resize stays usable until it is released."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
from vllm.v1.worker.workspace import WorkspaceManager
allocator = get_mem_allocator_instance()
manager = WorkspaceManager(
torch.device(DEVICE_TYPE),
alloc_context=lambda: allocator.use_memory_pool("workspace"),
)
(small,) = manager.get_simultaneous(((32 << 20,), torch.uint8))
# Resize while a view of the old buffer is alive: the tag is re-entered.
(scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8))
assert len(allocator.allocator_and_pools["workspace"]) == 2
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
scratch.add_(1)
backend = CuMemBackend()
backend.suspend(level=1)
assert mapped_usage(allocator) == 0
# Discarded, not copied.
assert all(
d.tag == "workspace" and d.cpu_backup_tensor is None
for d in allocator.pointer_to_data.values()
)
backend.resume(tags=["kv_cache"])
assert mapped_usage(allocator) == small.nbytes + scratch.nbytes
scratch.zero_()
graph.replay()
assert int(scratch.sum()) == scratch.numel()
small.fill_(1)
# The next resize releases the outgrown buffer; wake does not remap it.
del small
(big,) = manager.get_simultaneous(((128 << 20,), torch.uint8))
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
backend.suspend(level=1)
backend.resume(tags=["weights"])
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_reentered_tag(free_x_early):
"""Enter a tag twice (two workspace ubatches): freeing X returns its memory
by the next entry, without touching Y or breaking sleep/wake."""
allocator = get_mem_allocator_instance()
nbytes = 64 << 20
with allocator.use_memory_pool("t"):
xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)]
if free_x_early:
xs.clear()
with allocator.use_memory_pool("t"):
y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)
xs.clear()
gc.collect()
with allocator.use_memory_pool("t"):
pass
assert mapped_usage(allocator) == nbytes
y.fill_(1)
allocator.sleep(offload_tags="t")
allocator.wake_up()
assert int(y.sum()) == y.numel()
allocator.release_pools()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") @pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag(): def test_level2_discards_ordinary_tensor_with_weights_tag():
+3
View File
@@ -13,6 +13,9 @@ from vllm.platforms import current_platform
# py_handle has type list[int] on ROCm and int otherwise # py_handle has type list[int] on ROCm and int otherwise
HandleType: TypeAlias = tuple[int, int, int, list[int] | int] HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
# Selective wake may defer only these; any wake restores every other tag.
DEFERRABLE_TAGS: tuple[str, ...] = ("weights", "kv_cache")
@dataclasses.dataclass @dataclasses.dataclass
class AllocationData: class AllocationData:
+20 -13
View File
@@ -16,7 +16,7 @@ from typing import Any
import torch import torch
from vllm.device_allocator import AllocationData, HandleType from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
from vllm.device_allocator.alloc_conf import ( from vllm.device_allocator.alloc_conf import (
EXPANDABLE_SEGMENTS, EXPANDABLE_SEGMENTS,
conf_flag_enabled, conf_flag_enabled,
@@ -139,7 +139,7 @@ class CuMemAllocator:
def __init__(self): def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {} self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = CuMemAllocator.default_tag self.current_tag: str = CuMemAllocator.default_tag
self.allocator_and_pools: dict[str, Any] = {} self.allocator_and_pools: dict[str, list[Any]] = {}
# Creating strong references to the two callbacks here to prevent # Creating strong references to the two callbacks here to prevent
# these ephemeral bound-method objects being garbage collected. # these ephemeral bound-method objects being garbage collected.
# See discussions in https://github.com/vllm-project/vllm/pull/22724 # See discussions in https://github.com/vllm-project/vllm/pull/22724
@@ -167,7 +167,7 @@ class CuMemAllocator:
if not self.allocator_and_pools: if not self.allocator_and_pools:
return return
pool_entries = list(self.allocator_and_pools.values()) pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
self.allocator_and_pools.clear() self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries] mem_pools = [entry[0] for entry in pool_entries]
@@ -334,9 +334,9 @@ class CuMemAllocator:
memory, and the rest of the data will have empty memory. memory, and the rest of the data will have empty memory.
Args: Args:
tags: The tags of the memory allocation that will be loaded tags: The deferrable tags (weights, kv_cache) to load back to GPU
back to GPU memory. If None, all memory allocation will be loaded memory; every other tag is always loaded back. If None, all
back to GPU memory. memory allocation will be loaded back to GPU memory.
""" """
gc.collect() gc.collect()
@@ -345,7 +345,7 @@ class CuMemAllocator:
for ptr, data in self.pointer_to_data.items(): for ptr, data in self.pointer_to_data.items():
if not data.is_asleep: if not data.is_asleep:
continue continue
if tags is None or data.tag in tags: if tags is None or data.tag in tags or data.tag not in DEFERRABLE_TAGS:
handle = data.handle handle = data.handle
create_and_map(handle) create_and_map(handle)
data.is_asleep = False data.is_asleep = False
@@ -392,6 +392,10 @@ class CuMemAllocator:
old_tag = self.current_tag old_tag = self.current_tag
self.current_tag = tag self.current_tag = tag
try: try:
if tag != old_tag:
# Older pools of this tag are never reused, so trim them.
for pool, _ in self.allocator_and_pools.get(tag, []):
self._trim(pool)
with use_memory_pool_with_allocator( with use_memory_pool_with_allocator(
self.python_malloc_callback, self.python_free_callback self.python_malloc_callback, self.python_free_callback
) as data: ) as data:
@@ -400,7 +404,7 @@ class CuMemAllocator:
# and the memory pool. # and the memory pool.
# to avoid the issue, we keep a reference of the data. # to avoid the issue, we keep a reference of the data.
# see https://github.com/pytorch/pytorch/issues/146431 . # see https://github.com/pytorch/pytorch/issues/146431 .
self.allocator_and_pools[tag] = data self.allocator_and_pools.setdefault(tag, []).append(data)
yield yield
# PyTorch's bug, calling torch.cuda.empty_cache() will error # PyTorch's bug, calling torch.cuda.empty_cache() will error
# when using pluggable allocator, see # when using pluggable allocator, see
@@ -413,16 +417,19 @@ class CuMemAllocator:
# TODO: we should expose `empty_cache` method in the memory # TODO: we should expose `empty_cache` method in the memory
# pool. # pool.
# TODO: ask for help from PyTorch team to expose this method. # TODO: ask for help from PyTorch team to expose this method.
allocations = data[0].snapshot() self._trim(data[0])
for allocation in allocations:
if allocation["allocated_size"] == 0:
handle = self._python_free_callback(allocation["address"])
unmap_and_release(handle)
finally: finally:
self.current_tag = old_tag self.current_tag = old_tag
if expandable_was_enabled: if expandable_was_enabled:
set_alloc_conf(prev_conf) set_alloc_conf(prev_conf)
def _trim(self, pool: Any) -> None:
"""Release free segments of a pool that is never allocated from again."""
for allocation in pool.snapshot():
data = self.pointer_to_data.get(allocation["address"])
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
unmap_and_release(self._python_free_callback(allocation["address"]))
def get_current_usage(self) -> int: def get_current_usage(self) -> int:
"""Get the total number of bytes allocated in the memory pool.""" """Get the total number of bytes allocated in the memory pool."""
sum_bytes: int = 0 sum_bytes: int = 0
+10 -5
View File
@@ -9,7 +9,7 @@ from typing import Any
import torch import torch
from vllm.device_allocator import AllocationData, HandleType from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.torch_utils import PIN_MEMORY from vllm.utils.torch_utils import PIN_MEMORY
@@ -145,7 +145,7 @@ class XpuMemAllocator:
def __init__(self): def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {} self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = XpuMemAllocator.default_tag self.current_tag: str = XpuMemAllocator.default_tag
self.allocator_and_pools: dict[str, Any] = {} self.allocator_and_pools: dict[str, list[Any]] = {}
self.python_malloc_callback = self._python_malloc_callback self.python_malloc_callback = self._python_malloc_callback
self.python_free_callback = self._python_free_callback self.python_free_callback = self._python_free_callback
@@ -267,7 +267,11 @@ class XpuMemAllocator:
for ptr, data in self.pointer_to_data.items(): for ptr, data in self.pointer_to_data.items():
if not data.is_asleep: if not data.is_asleep:
continue continue
if tags is not None and data.tag not in tags: if (
tags is not None
and data.tag not in tags
and data.tag in DEFERRABLE_TAGS
):
continue continue
create_and_allocate(data.handle) create_and_allocate(data.handle)
data.is_asleep = False data.is_asleep = False
@@ -300,7 +304,7 @@ class XpuMemAllocator:
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete) # MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
# when releasing cached blocks. If allocator wrappers are dropped first, # when releasing cached blocks. If allocator wrappers are dropped first,
# C++ can hit "pure virtual method called" during shutdown. # C++ can hit "pure virtual method called" during shutdown.
pool_entries = list(self.allocator_and_pools.values()) pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
self.allocator_and_pools.clear() self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries] mem_pools = [entry[0] for entry in pool_entries]
@@ -333,7 +337,8 @@ class XpuMemAllocator:
self.python_malloc_callback, self.python_malloc_callback,
self.python_free_callback, self.python_free_callback,
) as data: ) as data:
self.allocator_and_pools[tag] = data # Keep every entry's pool alive (pytorch/pytorch#146431).
self.allocator_and_pools.setdefault(tag, []).append(data)
yield yield
finally: finally:
self.current_tag = old_tag self.current_tag = old_tag
+2
View File
@@ -532,6 +532,8 @@ class Worker(WorkerBase):
self.device, self.device,
num_ubatches, num_ubatches,
_num_workspace_lanes(self.vllm_config, self.use_v2_model_runner), _num_workspace_lanes(self.vllm_config, self.use_v2_model_runner),
# Scratch holds no state across steps: discard it on sleep.
alloc_context=lambda: self._maybe_get_memory_pool_context("workspace"),
) )
self.model_runner: GPUModelRunner = self._make_model_runner() self.model_runner: GPUModelRunner = self._make_model_runner()
if self.rank == 0: if self.rank == 0:
+11 -5
View File
@@ -4,7 +4,7 @@
import inspect import inspect
import os import os
from collections.abc import Callable, Hashable, Iterator from collections.abc import Callable, Hashable, Iterator
from contextlib import contextmanager from contextlib import AbstractContextManager, contextmanager, nullcontext
from contextvars import ContextVar from contextvars import ContextVar
from itertools import accumulate from itertools import accumulate
from math import prod from math import prod
@@ -58,8 +58,10 @@ class WorkspaceManager:
device: torch.device, device: torch.device,
num_ubatches: int | None = None, num_ubatches: int | None = None,
num_lanes: int = 1, num_lanes: int = 1,
alloc_context: Callable[[], AbstractContextManager] = nullcontext,
): ):
self._device = device self._device = device
self._alloc_context = alloc_context
# Cache num ubatches at init based on configuration (default to 1) # Cache num ubatches at init based on configuration (default to 1)
self._num_ubatches = num_ubatches if num_ubatches is not None else 1 self._num_ubatches = num_ubatches if num_ubatches is not None else 1
if num_lanes < 1: if num_lanes < 1:
@@ -261,9 +263,11 @@ class WorkspaceManager:
# dead segment in reserved memory which can cause higher peak # dead segment in reserved memory which can cause higher peak
# memory usage. # memory usage.
torch.accelerator.empty_cache() torch.accelerator.empty_cache()
self._current_workspaces[workspace_id] = torch.empty( # In sleep mode, entering the pool also trims outgrown buffers.
(required_bytes,), dtype=torch.uint8, device=self._device with self._alloc_context():
) self._current_workspaces[workspace_id] = torch.empty(
(required_bytes,), dtype=torch.uint8, device=self._device
)
current_workspace = self._current_workspaces[workspace_id] current_workspace = self._current_workspaces[workspace_id]
if envs.VLLM_DEBUG_WORKSPACE: if envs.VLLM_DEBUG_WORKSPACE:
@@ -308,6 +312,7 @@ def init_workspace_manager(
device: torch.device, device: torch.device,
num_ubatches: int | None = None, num_ubatches: int | None = None,
num_lanes: int = 1, num_lanes: int = 1,
alloc_context: Callable[[], AbstractContextManager] = nullcontext,
) -> None: ) -> None:
"""Initialize the workspace manager with a device. """Initialize the workspace manager with a device.
@@ -318,6 +323,7 @@ def init_workspace_manager(
device: The device to allocate workspace on. device: The device to allocate workspace on.
num_ubatches: Number of workspace ubatch slots. Defaults to 1. num_ubatches: Number of workspace ubatch slots. Defaults to 1.
num_lanes: Number of independent execution lanes per ubatch. Defaults to 1. num_lanes: Number of independent execution lanes per ubatch. Defaults to 1.
alloc_context: Factory for the context scratch is allocated in.
""" """
global _manager global _manager
@@ -328,7 +334,7 @@ def init_workspace_manager(
_manager._device, _manager._device,
device, device,
) )
_manager = WorkspaceManager(device, num_ubatches, num_lanes) _manager = WorkspaceManager(device, num_ubatches, num_lanes, alloc_context)
def lock_workspace() -> None: def lock_workspace() -> None: