diff --git a/.buildkite/intel_jobs/basic_correctness.yaml b/.buildkite/intel_jobs/basic_correctness.yaml new file mode 100644 index 0000000000..1b67454d2a --- /dev/null +++ b/.buildkite/intel_jobs/basic_correctness.yaml @@ -0,0 +1,22 @@ +group: Basic Correctness +depends_on: + - image-build-xpu +steps: +- label: XPU Sleep Mode + timeout_in_minutes: 30 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/test_cumem.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + pytest -v -s basic_correctness/test_mem.py::test_end_to_end' diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 97fc9c2bb9..1641300b06 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -908,10 +908,10 @@ steps: - vllm/ - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload - - tests/basic_correctness/test_cumem.py + - tests/basic_correctness/test_mem.py commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s basic_correctness/test_cumem.py + - pytest -v -s basic_correctness/test_mem.py - pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 5d547cd486..0310945b08 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -10,9 +10,9 @@ steps: - vllm/ - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload - - tests/basic_correctness/test_cumem.py + - tests/basic_correctness/test_mem.py commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s basic_correctness/test_cumem.py + - pytest -v -s basic_correctness/test_mem.py - pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py diff --git a/tests/basic_correctness/test_cumem.py b/tests/basic_correctness/test_mem.py similarity index 85% rename from tests/basic_correctness/test_cumem.py rename to tests/basic_correctness/test_mem.py index 8d8f87f0a3..2c9a99c500 100644 --- a/tests/basic_correctness/test_cumem.py +++ b/tests/basic_correctness/test_mem.py @@ -7,7 +7,7 @@ import pytest import torch from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams -from vllm.device_allocator.cumem import CuMemAllocator +from vllm.device_allocator import get_mem_allocator_instance from vllm.platforms import current_platform from vllm.utils.mem_constants import GiB_bytes @@ -16,14 +16,14 @@ from ..utils import create_new_process_for_each_test, requires_fp8 DEVICE_TYPE = current_platform.device_type -@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn") +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_python_error(): """ Test if Python error occurs when there's low-level error happening from the C++ side. """ - allocator = CuMemAllocator.get_instance() - total_bytes = torch.cuda.mem_get_info()[1] + allocator = get_mem_allocator_instance() + total_bytes = current_platform.mem_get_info()[1] alloc_bytes = int(total_bytes * 0.7) tensors = [] with allocator.use_memory_pool(): @@ -42,7 +42,7 @@ def test_python_error(): allocator.wake_up() -@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn") +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_basic_cumem(): # some tensors from default memory pool shape = (1024, 1024) @@ -50,7 +50,7 @@ def test_basic_cumem(): x.zero_() # some tensors from custom memory pool - allocator = CuMemAllocator.get_instance() + allocator = get_mem_allocator_instance() with allocator.use_memory_pool(): # custom memory pool y = torch.empty(shape, device=DEVICE_TYPE) @@ -64,9 +64,9 @@ def test_basic_cumem(): output = x + y + z assert torch.allclose(output, torch.ones_like(output) * 3) - free_bytes = torch.cuda.mem_get_info()[0] + free_bytes = current_platform.mem_get_info()[0] allocator.sleep() - free_bytes_after_sleep = torch.cuda.mem_get_info()[0] + free_bytes_after_sleep = current_platform.mem_get_info()[0] assert free_bytes_after_sleep > free_bytes allocator.wake_up() @@ -75,9 +75,10 @@ def test_basic_cumem(): assert torch.allclose(output, torch.ones_like(output) * 3) -@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn") +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU") def test_cumem_with_cudagraph(): - allocator = CuMemAllocator.get_instance() + allocator = get_mem_allocator_instance() with allocator.use_memory_pool(): weight = torch.eye(1024, device=DEVICE_TYPE) with allocator.use_memory_pool(tag="discard"): @@ -98,9 +99,9 @@ def test_cumem_with_cudagraph(): with torch.cuda.graph(model_graph): y = model(x) - free_bytes = torch.cuda.mem_get_info()[0] + free_bytes = current_platform.mem_get_info()[0] allocator.sleep() - free_bytes_after_sleep = torch.cuda.mem_get_info()[0] + free_bytes_after_sleep = current_platform.mem_get_info()[0] assert free_bytes_after_sleep > free_bytes allocator.wake_up() @@ -120,7 +121,7 @@ def test_cumem_with_cudagraph(): assert torch.allclose(y, x + 1) -@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn") +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @pytest.mark.parametrize( "model", [ @@ -131,7 +132,7 @@ def test_cumem_with_cudagraph(): ], ) def test_end_to_end(model: str): - free, total = torch.cuda.mem_get_info() + free, total = current_platform.mem_get_info() used_bytes_baseline = total - free # in case other process is running llm = LLM(model, enable_sleep_mode=True) prompt = "How are you?" @@ -143,7 +144,7 @@ def test_end_to_end(model: str): # test sleep level 1 here. llm.sleep(level=1) - free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info() + free_gpu_bytes_after_sleep, total = current_platform.mem_get_info() used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline # now the memory usage is mostly cudagraph memory pool, # and it should be less than the model weights (1B model, 2GiB weights) @@ -163,7 +164,7 @@ def test_end_to_end(model: str): llm.sleep(level=1) llm.wake_up(tags=["weights"]) - free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info() + free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info() used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline # should just reallocate memory for weights (1B model, ~2GiB weights) @@ -180,7 +181,7 @@ def test_end_to_end(model: str): @create_new_process_for_each_test() def test_deep_sleep(): model = "hmellor/tiny-random-LlamaForCausalLM" - free, total = torch.cuda.mem_get_info() + free, total = current_platform.mem_get_info() used_bytes_baseline = total - free # in case other process is running llm = LLM(model, enable_sleep_mode=True) prompt = "How are you?" @@ -190,13 +191,13 @@ def test_deep_sleep(): # Put the engine to deep sleep llm.sleep(level=2) - free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info() + free_gpu_bytes_after_sleep, total = current_platform.mem_get_info() used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline assert used_bytes < 3 * GiB_bytes llm.wake_up(tags=["weights"]) llm.collective_rpc("reload_weights") - free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info() + free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info() used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline assert used_bytes < 4 * GiB_bytes @@ -212,7 +213,7 @@ def test_deep_sleep(): def test_deep_sleep_async(): async def test(): model = "hmellor/tiny-random-LlamaForCausalLM" - free, total = torch.cuda.mem_get_info() + free, total = current_platform.mem_get_info() used_bytes_baseline = total - free # in case other process is running engine_args = AsyncEngineArgs( model=model, @@ -231,7 +232,7 @@ def test_deep_sleep_async(): await llm.wake_up(tags=["weights"]) await llm.collective_rpc("reload_weights") - free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info() + free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info() used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline assert used_bytes < 4 * GiB_bytes diff --git a/vllm/config/model.py b/vllm/config/model.py index f648a69e10..2f22140be9 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -528,7 +528,7 @@ class ModelConfig: if self.enable_sleep_mode: if not current_platform.is_sleep_mode_available(): raise ValueError("Sleep mode is not supported on current platform.") - if not self.enable_cumem_allocator: + if current_platform.is_cuda_alike() and not self.enable_cumem_allocator: logger.info_once( "Enabling cumem allocator because sleep mode requires it." ) diff --git a/vllm/device_allocator/__init__.py b/vllm/device_allocator/__init__.py index e69de29bb2..6b5e9c613d 100644 --- a/vllm/device_allocator/__init__.py +++ b/vllm/device_allocator/__init__.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import dataclasses +from contextlib import AbstractContextManager +from typing import Protocol + +import torch + +from vllm.platforms import current_platform + +# py_device, py_size_or_aligned_size, py_ptr, py_handle +HandleType = tuple[int, int, int, int] + + +@dataclasses.dataclass +class AllocationData: + handle: HandleType + tag: str + cpu_backup_tensor: torch.Tensor | None = None + + +class MemAllocator(Protocol): + def use_memory_pool(self, tag: str | None = None) -> AbstractContextManager: ... + + def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None: ... + + def wake_up(self, tags: list[str] | None = None) -> None: ... + + def get_current_usage(self) -> int: ... + + +def get_mem_allocator_instance() -> MemAllocator: + if current_platform.is_cuda_alike(): + from vllm.device_allocator.cumem import CuMemAllocator + + return CuMemAllocator.get_instance() + + if current_platform.is_xpu(): + from vllm.device_allocator.xpumem import XpuMemAllocator + + return XpuMemAllocator.get_instance() + + raise RuntimeError( + "Sleep mode allocator is not available on platform " + f"{type(current_platform).__name__} " + f"(device_type={current_platform.device_type})." + ) diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index a652c5a6c7..c30790df9c 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -8,7 +8,6 @@ # both of them failed because of cuda context mismatch. # not sure why, they are created from a different context. # the only successful approach is to call cuda driver API in C. -import dataclasses import gc import os from collections.abc import Callable, Iterator @@ -17,6 +16,7 @@ from typing import Any import torch +from vllm.device_allocator import AllocationData, HandleType from vllm.logger import init_logger from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.system_utils import find_loaded_library @@ -44,16 +44,6 @@ except ModuleNotFoundError: python_unmap_and_release = None lib_name = None -# py_device, py_alignedSize, py_d_mem, py_p_memHandle -HandleType = tuple[int, int, int, int] - - -@dataclasses.dataclass -class AllocationData: - handle: HandleType - tag: str - cpu_backup_tensor: torch.Tensor | None = None - def create_and_map(allocation_handle: HandleType) -> None: python_create_and_map(*allocation_handle) diff --git a/vllm/device_allocator/xpumem.py b/vllm/device_allocator/xpumem.py new file mode 100644 index 0000000000..7d99ced7ef --- /dev/null +++ b/vllm/device_allocator/xpumem.py @@ -0,0 +1,295 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import atexit +import gc +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +import torch + +from vllm.device_allocator import AllocationData, HandleType +from vllm.logger import init_logger +from vllm.utils.platform_utils import is_pin_memory_available + +logger = init_logger(__name__) + +MEMCPY_HOST_TO_DEVICE = 0 +MEMCPY_DEVICE_TO_HOST = 1 +MEMCPY_DEVICE_TO_DEVICE = 2 + +xpumem_available = False +xpumem_allocator: Any = None + +try: + from vllm_xpu_kernels import xpumem_allocator as _xpumem_allocator + + xpumem_allocator = _xpumem_allocator + xpumem_available = True +except ImportError: + xpumem_allocator = None + + +def _xpu_memory_module() -> Any: + mem_mod = getattr(torch.xpu, "memory", None) + if mem_mod is None: + raise RuntimeError("torch.xpu.memory is not available") + return mem_mod + + +def _supports_xpu_mem_pool(mem_mod: Any) -> bool: + return hasattr(mem_mod, "MemPool") and hasattr(mem_mod, "use_mem_pool") + + +def _xpu_memcpy_sync( + dst_ptr: int, + src_ptr: int, + n_bytes: int, + kind: int, + device: int, +) -> None: + def _to_i64_ptr(ptr: int) -> int: + # torch custom-op `int` arguments are signed int64. + # data_ptr() may return a uint64 value above 2^63-1, so normalize it. + return ptr if ptr < (1 << 63) else ptr - (1 << 64) + + torch.ops._C.xpu_memcpy_sync( + _to_i64_ptr(dst_ptr), + _to_i64_ptr(src_ptr), + n_bytes, + kind, + device, + ) + + +def get_pluggable_allocator( + python_malloc_fn: Callable[[HandleType], None], + python_free_func: Callable[[int], HandleType], +) -> Any: + if not xpumem_available or xpumem_allocator is None: + raise RuntimeError("xpumem allocator extension is not available") + + xpumem_allocator.init_module(python_malloc_fn, python_free_func) + mem_mod = _xpu_memory_module() + alloc_cls = getattr(mem_mod, "XPUPluggableAllocator", None) + if alloc_cls is None: + raise RuntimeError("torch.xpu.memory.XPUPluggableAllocator is not available") + + lib_name = xpumem_allocator.__file__ + return alloc_cls(lib_name, "my_malloc", "my_free") + + +def create_and_allocate(allocation_handle: HandleType) -> None: + if not xpumem_available or xpumem_allocator is None: + raise RuntimeError("xpumem allocator extension is not available") + xpumem_allocator.python_create_and_allocate(*allocation_handle) + + +def unmap_and_release(allocation_handle: HandleType) -> None: + if not xpumem_available or xpumem_allocator is None: + raise RuntimeError("xpumem allocator extension is not available") + xpumem_allocator.python_unmap_and_release(*allocation_handle) + + +@contextmanager +def use_memory_pool_with_allocator( + python_malloc_fn: Callable[[HandleType], None], + python_free_func: Callable[[int], HandleType], +) -> Iterator[tuple[Any, Any]]: + mem_mod = _xpu_memory_module() + if not _supports_xpu_mem_pool(mem_mod): + raise RuntimeError( + "torch.xpu.memory MemPool APIs are not available " + "(need MemPool and use_mem_pool)." + ) + new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func) + mem_pool = mem_mod.MemPool(new_alloc._allocator) + with mem_mod.use_mem_pool(mem_pool): + yield mem_pool, new_alloc + + +class XpuMemAllocator: + """A singleton pluggable allocator helper for XPU. + + Note: + Sleep will offload selected payloads to CPU or discard and unmap XPU + physical memory. Wake-up remaps physical memory back to the same + reserved virtual address and restores payload. + """ + + instance: "XpuMemAllocator | None" = None + default_tag: str = "default" + + @staticmethod + def get_instance() -> "XpuMemAllocator": + assert xpumem_available, "xpumem allocator is not available" + if XpuMemAllocator.instance is None: + XpuMemAllocator.instance = XpuMemAllocator() + # Ensure MemPool/allocator wrappers are released before interpreter + # finalization tears down XPU runtime internals. + atexit.register(XpuMemAllocator._shutdown_singleton) + return XpuMemAllocator.instance + + @staticmethod + def _shutdown_singleton() -> None: + instance = XpuMemAllocator.instance + if instance is None: + return + try: + instance.release_pools() + except Exception: + logger.exception("XpuMemAllocator singleton shutdown failed") + + def __init__(self): + self.pointer_to_data: dict[int, AllocationData] = {} + self.current_tag: str = XpuMemAllocator.default_tag + self.allocator_and_pools: dict[str, Any] = {} + self.python_malloc_callback = self._python_malloc_callback + self.python_free_callback = self._python_free_callback + + def _python_malloc_callback(self, allocation_handle: HandleType) -> None: + ptr = allocation_handle[2] + self.pointer_to_data[ptr] = AllocationData(allocation_handle, self.current_tag) + logger.debug( + "Allocated %s bytes for %s at %s", + allocation_handle[1], + self.current_tag, + ptr, + ) + + def _python_free_callback(self, ptr: int) -> HandleType: + data = self.pointer_to_data.pop(ptr) + data.cpu_backup_tensor = None + logger.debug("Freed %s bytes for %s at %s", data.handle[1], data.tag, ptr) + return data.handle + + def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None: + if offload_tags is None: + offload_tags = (XpuMemAllocator.default_tag,) + elif isinstance(offload_tags, str): + offload_tags = (offload_tags,) + + assert isinstance(offload_tags, tuple) + + total_bytes = 0 + backup_bytes = 0 + + for ptr, data in self.pointer_to_data.items(): + size_in_bytes = data.handle[1] + total_bytes += size_in_bytes + if data.tag not in offload_tags: + unmap_and_release(data.handle) + continue + + backup_bytes += size_in_bytes + device, _, _, _ = data.handle + cpu_backup_tensor = torch.empty( + size_in_bytes, + dtype=torch.uint8, + device="cpu", + pin_memory=is_pin_memory_available(), + ) + cpu_ptr = cpu_backup_tensor.data_ptr() + _xpu_memcpy_sync( + cpu_ptr, + ptr, + size_in_bytes, + MEMCPY_DEVICE_TO_HOST, + device, + ) + data.cpu_backup_tensor = cpu_backup_tensor + + unmap_and_release(data.handle) + + logger.info( + "XpuMemAllocator: sleep freed %.2f GiB memory in total, of which " + "%.2f GiB is backed up in CPU and the rest %.2f GiB is discarded " + "directly.", + total_bytes / 1024**3, + backup_bytes / 1024**3, + (total_bytes - backup_bytes) / 1024**3, + ) + + gc.collect() + xpu_empty_cache = getattr(torch.xpu, "empty_cache", None) + if callable(xpu_empty_cache): + xpu_empty_cache() + + def wake_up(self, tags: list[str] | None = None) -> None: + for ptr, data in self.pointer_to_data.items(): + if tags is not None and data.tag not in tags: + continue + create_and_allocate(data.handle) + + cpu_backup_tensor = data.cpu_backup_tensor + if cpu_backup_tensor is None: + continue + + device, size_in_bytes, _, _ = data.handle + _xpu_memcpy_sync( + ptr, + cpu_backup_tensor.data_ptr(), + size_in_bytes, + MEMCPY_HOST_TO_DEVICE, + device, + ) + data.cpu_backup_tensor = None + + def release_pools(self) -> None: + """Drop Python references to MemPool/pluggable allocators eagerly. + + This prevents pool destruction from being deferred to interpreter + finalization, which can happen after parts of XPU runtime are already + torn down. + """ + if not self.allocator_and_pools: + return + + # Note: keep allocators alive while MemPool objects are destroyed. + # MemPool teardown may invoke allocator virtual methods (e.g. raw_delete) + # when releasing cached blocks. If allocator wrappers are dropped first, + # C++ can hit "pure virtual method called" during shutdown. + pool_entries = list(self.allocator_and_pools.values()) + self.allocator_and_pools.clear() + + mem_pools = [entry[0] for entry in pool_entries] + allocators = [entry[1] for entry in pool_entries] + pool_entries.clear() + + xpu_sync = getattr(torch.xpu, "synchronize", None) + if callable(xpu_sync): + try: + xpu_sync() + except Exception: + logger.debug("torch.xpu.synchronize() failed during release_pools") + + # Phase 1: drop MemPool refs while allocators are still strongly held. + mem_pools.clear() + gc.collect() + + # Phase 2: now it is safe to release allocator wrappers. + allocators.clear() + + @contextmanager + def use_memory_pool(self, tag: str | None = None): + if tag is None: + tag = XpuMemAllocator.default_tag + + old_tag = self.current_tag + self.current_tag = tag + try: + with use_memory_pool_with_allocator( + self.python_malloc_callback, + self.python_free_callback, + ) as data: + self.allocator_and_pools[tag] = data + yield + finally: + self.current_tag = old_tag + + def get_current_usage(self) -> int: + total = 0 + for data in self.pointer_to_data.values(): + total += data.handle[1] + return total diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index b357c5798b..413234b402 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -197,7 +197,7 @@ class Platform: # for ROCm, but currently we don't have a way to detect the # exact GPU model statelessly here. So we return True for # all ROCm platforms for now. - return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM) + return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM, PlatformEnum.XPU) def is_cumem_allocator_available(self) -> bool: try: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index bf6b287d8d..052e1fe76f 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -18,6 +18,7 @@ import torch.nn as nn import vllm.envs as envs from vllm.config import CUDAGraphMode, VllmConfig, set_current_vllm_config from vllm.config.compilation import CompilationMode +from vllm.device_allocator import get_mem_allocator_instance from vllm.distributed import ( ensure_model_parallel_initialized, init_distributed_environment, @@ -162,8 +163,6 @@ class Worker(WorkerBase): self._pp_send_work: list[Handle] = [] def sleep(self, level: int = 1) -> None: - from vllm.device_allocator.cumem import CuMemAllocator - free_bytes_before_sleep = torch.cuda.mem_get_info()[0] # Save the buffers before level 2 sleep @@ -173,7 +172,7 @@ class Worker(WorkerBase): name: buffer.cpu().clone() for name, buffer in model.named_buffers() } - allocator = CuMemAllocator.get_instance() + allocator = get_mem_allocator_instance() allocator.sleep(offload_tags=("weights",) if level == 1 else tuple()) free_bytes_after_sleep, total = torch.cuda.mem_get_info() freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep @@ -186,9 +185,7 @@ class Worker(WorkerBase): ) def wake_up(self, tags: list[str] | None = None) -> None: - from vllm.device_allocator.cumem import CuMemAllocator - - allocator = CuMemAllocator.get_instance() + allocator = get_mem_allocator_instance() allocator.wake_up(tags) # Restore the buffers after level 2 sleep @@ -203,12 +200,22 @@ class Worker(WorkerBase): self.model_runner.post_kv_cache_wake_up() def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager: - if not self.vllm_config.model_config.enable_cumem_allocator: + if ( + current_platform.is_cuda_alike() + and not self.vllm_config.model_config.enable_cumem_allocator + ): return nullcontext() - from vllm.device_allocator.cumem import CuMemAllocator + if ( + current_platform.is_xpu() + and not self.vllm_config.model_config.enable_sleep_mode + ): + return nullcontext() - allocator = CuMemAllocator.get_instance() + if current_platform.is_cpu(): + return nullcontext() + + allocator = get_mem_allocator_instance() if tag == "weights": assert allocator.get_current_usage() == 0, ( "CuMem allocator can only be used for one instance per process."