[XPU][Feature] transparent sleep mode support for XPU platform (#37149)
Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
group: Basic Correctness
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: XPU Sleep Mode
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/test_cumem.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
|
||||
@@ -908,10 +908,10 @@ steps:
|
||||
- vllm/
|
||||
- tests/basic_correctness/test_basic_correctness
|
||||
- tests/basic_correctness/test_cpu_offload
|
||||
- tests/basic_correctness/test_cumem.py
|
||||
- tests/basic_correctness/test_mem.py
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/test_cumem.py
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
|
||||
|
||||
@@ -10,9 +10,9 @@ steps:
|
||||
- vllm/
|
||||
- tests/basic_correctness/test_basic_correctness
|
||||
- tests/basic_correctness/test_cpu_offload
|
||||
- tests/basic_correctness/test_cumem.py
|
||||
- tests/basic_correctness/test_mem.py
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/test_cumem.py
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
@@ -16,14 +16,14 @@ from ..utils import create_new_process_for_each_test, requires_fp8
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_python_error():
|
||||
"""
|
||||
Test if Python error occurs when there's low-level
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
total_bytes = torch.cuda.mem_get_info()[1]
|
||||
allocator = get_mem_allocator_instance()
|
||||
total_bytes = current_platform.mem_get_info()[1]
|
||||
alloc_bytes = int(total_bytes * 0.7)
|
||||
tensors = []
|
||||
with allocator.use_memory_pool():
|
||||
@@ -42,7 +42,7 @@ def test_python_error():
|
||||
allocator.wake_up()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_basic_cumem():
|
||||
# some tensors from default memory pool
|
||||
shape = (1024, 1024)
|
||||
@@ -50,7 +50,7 @@ def test_basic_cumem():
|
||||
x.zero_()
|
||||
|
||||
# some tensors from custom memory pool
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
# custom memory pool
|
||||
y = torch.empty(shape, device=DEVICE_TYPE)
|
||||
@@ -64,9 +64,9 @@ def test_basic_cumem():
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
free_bytes = torch.cuda.mem_get_info()[0]
|
||||
free_bytes = current_platform.mem_get_info()[0]
|
||||
allocator.sleep()
|
||||
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
|
||||
free_bytes_after_sleep = current_platform.mem_get_info()[0]
|
||||
assert free_bytes_after_sleep > free_bytes
|
||||
allocator.wake_up()
|
||||
|
||||
@@ -75,9 +75,10 @@ def test_basic_cumem():
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
||||
def test_cumem_with_cudagraph():
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
weight = torch.eye(1024, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool(tag="discard"):
|
||||
@@ -98,9 +99,9 @@ def test_cumem_with_cudagraph():
|
||||
with torch.cuda.graph(model_graph):
|
||||
y = model(x)
|
||||
|
||||
free_bytes = torch.cuda.mem_get_info()[0]
|
||||
free_bytes = current_platform.mem_get_info()[0]
|
||||
allocator.sleep()
|
||||
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
|
||||
free_bytes_after_sleep = current_platform.mem_get_info()[0]
|
||||
assert free_bytes_after_sleep > free_bytes
|
||||
allocator.wake_up()
|
||||
|
||||
@@ -120,7 +121,7 @@ def test_cumem_with_cudagraph():
|
||||
assert torch.allclose(y, x + 1)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
@@ -131,7 +132,7 @@ def test_cumem_with_cudagraph():
|
||||
],
|
||||
)
|
||||
def test_end_to_end(model: str):
|
||||
free, total = torch.cuda.mem_get_info()
|
||||
free, total = current_platform.mem_get_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
@@ -143,7 +144,7 @@ def test_end_to_end(model: str):
|
||||
# test sleep level 1 here.
|
||||
llm.sleep(level=1)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
# now the memory usage is mostly cudagraph memory pool,
|
||||
# and it should be less than the model weights (1B model, 2GiB weights)
|
||||
@@ -163,7 +164,7 @@ def test_end_to_end(model: str):
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up(tags=["weights"])
|
||||
|
||||
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
|
||||
# should just reallocate memory for weights (1B model, ~2GiB weights)
|
||||
@@ -180,7 +181,7 @@ def test_end_to_end(model: str):
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.cuda.mem_get_info()
|
||||
free, total = current_platform.mem_get_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
@@ -190,13 +191,13 @@ def test_deep_sleep():
|
||||
# Put the engine to deep sleep
|
||||
llm.sleep(level=2)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
assert used_bytes < 3 * GiB_bytes
|
||||
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
@@ -212,7 +213,7 @@ def test_deep_sleep():
|
||||
def test_deep_sleep_async():
|
||||
async def test():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.cuda.mem_get_info()
|
||||
free, total = current_platform.mem_get_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=model,
|
||||
@@ -231,7 +232,7 @@ def test_deep_sleep_async():
|
||||
|
||||
await llm.wake_up(tags=["weights"])
|
||||
await llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
@@ -528,7 +528,7 @@ class ModelConfig:
|
||||
if self.enable_sleep_mode:
|
||||
if not current_platform.is_sleep_mode_available():
|
||||
raise ValueError("Sleep mode is not supported on current platform.")
|
||||
if not self.enable_cumem_allocator:
|
||||
if current_platform.is_cuda_alike() and not self.enable_cumem_allocator:
|
||||
logger.info_once(
|
||||
"Enabling cumem allocator because sleep mode requires it."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import dataclasses
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# py_device, py_size_or_aligned_size, py_ptr, py_handle
|
||||
HandleType = tuple[int, int, int, int]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class AllocationData:
|
||||
handle: HandleType
|
||||
tag: str
|
||||
cpu_backup_tensor: torch.Tensor | None = None
|
||||
|
||||
|
||||
class MemAllocator(Protocol):
|
||||
def use_memory_pool(self, tag: str | None = None) -> AbstractContextManager: ...
|
||||
|
||||
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None: ...
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None: ...
|
||||
|
||||
def get_current_usage(self) -> int: ...
|
||||
|
||||
|
||||
def get_mem_allocator_instance() -> MemAllocator:
|
||||
if current_platform.is_cuda_alike():
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
return CuMemAllocator.get_instance()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm.device_allocator.xpumem import XpuMemAllocator
|
||||
|
||||
return XpuMemAllocator.get_instance()
|
||||
|
||||
raise RuntimeError(
|
||||
"Sleep mode allocator is not available on platform "
|
||||
f"{type(current_platform).__name__} "
|
||||
f"(device_type={current_platform.device_type})."
|
||||
)
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
# both of them failed because of cuda context mismatch.
|
||||
# not sure why, they are created from a different context.
|
||||
# the only successful approach is to call cuda driver API in C.
|
||||
import dataclasses
|
||||
import gc
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
@@ -17,6 +16,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.system_utils import find_loaded_library
|
||||
@@ -44,16 +44,6 @@ except ModuleNotFoundError:
|
||||
python_unmap_and_release = None
|
||||
lib_name = None
|
||||
|
||||
# py_device, py_alignedSize, py_d_mem, py_p_memHandle
|
||||
HandleType = tuple[int, int, int, int]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class AllocationData:
|
||||
handle: HandleType
|
||||
tag: str
|
||||
cpu_backup_tensor: torch.Tensor | None = None
|
||||
|
||||
|
||||
def create_and_map(allocation_handle: HandleType) -> None:
|
||||
python_create_and_map(*allocation_handle)
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import atexit
|
||||
import gc
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
MEMCPY_HOST_TO_DEVICE = 0
|
||||
MEMCPY_DEVICE_TO_HOST = 1
|
||||
MEMCPY_DEVICE_TO_DEVICE = 2
|
||||
|
||||
xpumem_available = False
|
||||
xpumem_allocator: Any = None
|
||||
|
||||
try:
|
||||
from vllm_xpu_kernels import xpumem_allocator as _xpumem_allocator
|
||||
|
||||
xpumem_allocator = _xpumem_allocator
|
||||
xpumem_available = True
|
||||
except ImportError:
|
||||
xpumem_allocator = None
|
||||
|
||||
|
||||
def _xpu_memory_module() -> Any:
|
||||
mem_mod = getattr(torch.xpu, "memory", None)
|
||||
if mem_mod is None:
|
||||
raise RuntimeError("torch.xpu.memory is not available")
|
||||
return mem_mod
|
||||
|
||||
|
||||
def _supports_xpu_mem_pool(mem_mod: Any) -> bool:
|
||||
return hasattr(mem_mod, "MemPool") and hasattr(mem_mod, "use_mem_pool")
|
||||
|
||||
|
||||
def _xpu_memcpy_sync(
|
||||
dst_ptr: int,
|
||||
src_ptr: int,
|
||||
n_bytes: int,
|
||||
kind: int,
|
||||
device: int,
|
||||
) -> None:
|
||||
def _to_i64_ptr(ptr: int) -> int:
|
||||
# torch custom-op `int` arguments are signed int64.
|
||||
# data_ptr() may return a uint64 value above 2^63-1, so normalize it.
|
||||
return ptr if ptr < (1 << 63) else ptr - (1 << 64)
|
||||
|
||||
torch.ops._C.xpu_memcpy_sync(
|
||||
_to_i64_ptr(dst_ptr),
|
||||
_to_i64_ptr(src_ptr),
|
||||
n_bytes,
|
||||
kind,
|
||||
device,
|
||||
)
|
||||
|
||||
|
||||
def get_pluggable_allocator(
|
||||
python_malloc_fn: Callable[[HandleType], None],
|
||||
python_free_func: Callable[[int], HandleType],
|
||||
) -> Any:
|
||||
if not xpumem_available or xpumem_allocator is None:
|
||||
raise RuntimeError("xpumem allocator extension is not available")
|
||||
|
||||
xpumem_allocator.init_module(python_malloc_fn, python_free_func)
|
||||
mem_mod = _xpu_memory_module()
|
||||
alloc_cls = getattr(mem_mod, "XPUPluggableAllocator", None)
|
||||
if alloc_cls is None:
|
||||
raise RuntimeError("torch.xpu.memory.XPUPluggableAllocator is not available")
|
||||
|
||||
lib_name = xpumem_allocator.__file__
|
||||
return alloc_cls(lib_name, "my_malloc", "my_free")
|
||||
|
||||
|
||||
def create_and_allocate(allocation_handle: HandleType) -> None:
|
||||
if not xpumem_available or xpumem_allocator is None:
|
||||
raise RuntimeError("xpumem allocator extension is not available")
|
||||
xpumem_allocator.python_create_and_allocate(*allocation_handle)
|
||||
|
||||
|
||||
def unmap_and_release(allocation_handle: HandleType) -> None:
|
||||
if not xpumem_available or xpumem_allocator is None:
|
||||
raise RuntimeError("xpumem allocator extension is not available")
|
||||
xpumem_allocator.python_unmap_and_release(*allocation_handle)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_memory_pool_with_allocator(
|
||||
python_malloc_fn: Callable[[HandleType], None],
|
||||
python_free_func: Callable[[int], HandleType],
|
||||
) -> Iterator[tuple[Any, Any]]:
|
||||
mem_mod = _xpu_memory_module()
|
||||
if not _supports_xpu_mem_pool(mem_mod):
|
||||
raise RuntimeError(
|
||||
"torch.xpu.memory MemPool APIs are not available "
|
||||
"(need MemPool and use_mem_pool)."
|
||||
)
|
||||
new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func)
|
||||
mem_pool = mem_mod.MemPool(new_alloc._allocator)
|
||||
with mem_mod.use_mem_pool(mem_pool):
|
||||
yield mem_pool, new_alloc
|
||||
|
||||
|
||||
class XpuMemAllocator:
|
||||
"""A singleton pluggable allocator helper for XPU.
|
||||
|
||||
Note:
|
||||
Sleep will offload selected payloads to CPU or discard and unmap XPU
|
||||
physical memory. Wake-up remaps physical memory back to the same
|
||||
reserved virtual address and restores payload.
|
||||
"""
|
||||
|
||||
instance: "XpuMemAllocator | None" = None
|
||||
default_tag: str = "default"
|
||||
|
||||
@staticmethod
|
||||
def get_instance() -> "XpuMemAllocator":
|
||||
assert xpumem_available, "xpumem allocator is not available"
|
||||
if XpuMemAllocator.instance is None:
|
||||
XpuMemAllocator.instance = XpuMemAllocator()
|
||||
# Ensure MemPool/allocator wrappers are released before interpreter
|
||||
# finalization tears down XPU runtime internals.
|
||||
atexit.register(XpuMemAllocator._shutdown_singleton)
|
||||
return XpuMemAllocator.instance
|
||||
|
||||
@staticmethod
|
||||
def _shutdown_singleton() -> None:
|
||||
instance = XpuMemAllocator.instance
|
||||
if instance is None:
|
||||
return
|
||||
try:
|
||||
instance.release_pools()
|
||||
except Exception:
|
||||
logger.exception("XpuMemAllocator singleton shutdown failed")
|
||||
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = XpuMemAllocator.default_tag
|
||||
self.allocator_and_pools: dict[str, Any] = {}
|
||||
self.python_malloc_callback = self._python_malloc_callback
|
||||
self.python_free_callback = self._python_free_callback
|
||||
|
||||
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
|
||||
ptr = allocation_handle[2]
|
||||
self.pointer_to_data[ptr] = AllocationData(allocation_handle, self.current_tag)
|
||||
logger.debug(
|
||||
"Allocated %s bytes for %s at %s",
|
||||
allocation_handle[1],
|
||||
self.current_tag,
|
||||
ptr,
|
||||
)
|
||||
|
||||
def _python_free_callback(self, ptr: int) -> HandleType:
|
||||
data = self.pointer_to_data.pop(ptr)
|
||||
data.cpu_backup_tensor = None
|
||||
logger.debug("Freed %s bytes for %s at %s", data.handle[1], data.tag, ptr)
|
||||
return data.handle
|
||||
|
||||
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
|
||||
if offload_tags is None:
|
||||
offload_tags = (XpuMemAllocator.default_tag,)
|
||||
elif isinstance(offload_tags, str):
|
||||
offload_tags = (offload_tags,)
|
||||
|
||||
assert isinstance(offload_tags, tuple)
|
||||
|
||||
total_bytes = 0
|
||||
backup_bytes = 0
|
||||
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
size_in_bytes = data.handle[1]
|
||||
total_bytes += size_in_bytes
|
||||
if data.tag not in offload_tags:
|
||||
unmap_and_release(data.handle)
|
||||
continue
|
||||
|
||||
backup_bytes += size_in_bytes
|
||||
device, _, _, _ = data.handle
|
||||
cpu_backup_tensor = torch.empty(
|
||||
size_in_bytes,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=is_pin_memory_available(),
|
||||
)
|
||||
cpu_ptr = cpu_backup_tensor.data_ptr()
|
||||
_xpu_memcpy_sync(
|
||||
cpu_ptr,
|
||||
ptr,
|
||||
size_in_bytes,
|
||||
MEMCPY_DEVICE_TO_HOST,
|
||||
device,
|
||||
)
|
||||
data.cpu_backup_tensor = cpu_backup_tensor
|
||||
|
||||
unmap_and_release(data.handle)
|
||||
|
||||
logger.info(
|
||||
"XpuMemAllocator: sleep freed %.2f GiB memory in total, of which "
|
||||
"%.2f GiB is backed up in CPU and the rest %.2f GiB is discarded "
|
||||
"directly.",
|
||||
total_bytes / 1024**3,
|
||||
backup_bytes / 1024**3,
|
||||
(total_bytes - backup_bytes) / 1024**3,
|
||||
)
|
||||
|
||||
gc.collect()
|
||||
xpu_empty_cache = getattr(torch.xpu, "empty_cache", None)
|
||||
if callable(xpu_empty_cache):
|
||||
xpu_empty_cache()
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
if tags is not None and data.tag not in tags:
|
||||
continue
|
||||
create_and_allocate(data.handle)
|
||||
|
||||
cpu_backup_tensor = data.cpu_backup_tensor
|
||||
if cpu_backup_tensor is None:
|
||||
continue
|
||||
|
||||
device, size_in_bytes, _, _ = data.handle
|
||||
_xpu_memcpy_sync(
|
||||
ptr,
|
||||
cpu_backup_tensor.data_ptr(),
|
||||
size_in_bytes,
|
||||
MEMCPY_HOST_TO_DEVICE,
|
||||
device,
|
||||
)
|
||||
data.cpu_backup_tensor = None
|
||||
|
||||
def release_pools(self) -> None:
|
||||
"""Drop Python references to MemPool/pluggable allocators eagerly.
|
||||
|
||||
This prevents pool destruction from being deferred to interpreter
|
||||
finalization, which can happen after parts of XPU runtime are already
|
||||
torn down.
|
||||
"""
|
||||
if not self.allocator_and_pools:
|
||||
return
|
||||
|
||||
# Note: keep allocators alive while MemPool objects are destroyed.
|
||||
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
|
||||
# when releasing cached blocks. If allocator wrappers are dropped first,
|
||||
# C++ can hit "pure virtual method called" during shutdown.
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
allocators = [entry[1] for entry in pool_entries]
|
||||
pool_entries.clear()
|
||||
|
||||
xpu_sync = getattr(torch.xpu, "synchronize", None)
|
||||
if callable(xpu_sync):
|
||||
try:
|
||||
xpu_sync()
|
||||
except Exception:
|
||||
logger.debug("torch.xpu.synchronize() failed during release_pools")
|
||||
|
||||
# Phase 1: drop MemPool refs while allocators are still strongly held.
|
||||
mem_pools.clear()
|
||||
gc.collect()
|
||||
|
||||
# Phase 2: now it is safe to release allocator wrappers.
|
||||
allocators.clear()
|
||||
|
||||
@contextmanager
|
||||
def use_memory_pool(self, tag: str | None = None):
|
||||
if tag is None:
|
||||
tag = XpuMemAllocator.default_tag
|
||||
|
||||
old_tag = self.current_tag
|
||||
self.current_tag = tag
|
||||
try:
|
||||
with use_memory_pool_with_allocator(
|
||||
self.python_malloc_callback,
|
||||
self.python_free_callback,
|
||||
) as data:
|
||||
self.allocator_and_pools[tag] = data
|
||||
yield
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
|
||||
def get_current_usage(self) -> int:
|
||||
total = 0
|
||||
for data in self.pointer_to_data.values():
|
||||
total += data.handle[1]
|
||||
return total
|
||||
@@ -197,7 +197,7 @@ class Platform:
|
||||
# for ROCm, but currently we don't have a way to detect the
|
||||
# exact GPU model statelessly here. So we return True for
|
||||
# all ROCm platforms for now.
|
||||
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
|
||||
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM, PlatformEnum.XPU)
|
||||
|
||||
def is_cumem_allocator_available(self) -> bool:
|
||||
try:
|
||||
|
||||
@@ -18,6 +18,7 @@ import torch.nn as nn
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
@@ -162,8 +163,6 @@ class Worker(WorkerBase):
|
||||
self._pp_send_work: list[Handle] = []
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
@@ -173,7 +172,7 @@ class Worker(WorkerBase):
|
||||
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
|
||||
}
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
@@ -186,9 +185,7 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator = get_mem_allocator_instance()
|
||||
allocator.wake_up(tags)
|
||||
|
||||
# Restore the buffers after level 2 sleep
|
||||
@@ -203,12 +200,22 @@ class Worker(WorkerBase):
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if not self.vllm_config.model_config.enable_cumem_allocator:
|
||||
if (
|
||||
current_platform.is_cuda_alike()
|
||||
and not self.vllm_config.model_config.enable_cumem_allocator
|
||||
):
|
||||
return nullcontext()
|
||||
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
if (
|
||||
current_platform.is_xpu()
|
||||
and not self.vllm_config.model_config.enable_sleep_mode
|
||||
):
|
||||
return nullcontext()
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
if current_platform.is_cpu():
|
||||
return nullcontext()
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
if tag == "weights":
|
||||
assert allocator.get_current_usage() == 0, (
|
||||
"CuMem allocator can only be used for one instance per process."
|
||||
|
||||
Reference in New Issue
Block a user