[CPU] Experimentally enable Triton and MRV2 (#43225)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+20
-3
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.utils.cpu_resource_utils import (
|
||||
DEVICE_CONTROL_ENV_VAR,
|
||||
get_memory_node_info,
|
||||
get_visible_memory_node,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -135,9 +136,13 @@ class CpuPlatform(Platform):
|
||||
scheduler_config.async_scheduling = False
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# OMP requires the MP executor to function correctly, UniProc is not
|
||||
# supported as it is not possible to set the OMP environment correctly
|
||||
if parallel_config.distributed_executor_backend == "uni":
|
||||
if (
|
||||
os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING", "1") == "1"
|
||||
and parallel_config.distributed_executor_backend == "uni"
|
||||
):
|
||||
# OMP requires the MP executor to function correctly, UniProc
|
||||
# is not supported as it is not possible to set the OMP
|
||||
# environment correctly
|
||||
parallel_config.distributed_executor_backend = "mp"
|
||||
|
||||
if parallel_config.worker_cls == "auto":
|
||||
@@ -481,3 +486,15 @@ class CpuPlatform(Platform):
|
||||
slot_mapping,
|
||||
isa,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_current_memory_usage(
|
||||
cls, device: torch.types.Device | None = None
|
||||
) -> float:
|
||||
allowed_mem_node_list = get_visible_memory_node()
|
||||
mem_status_list = [get_memory_node_info(i) for i in allowed_mem_node_list]
|
||||
memory_usage = 0
|
||||
for s in mem_status_list:
|
||||
memory_usage += s.total_memory - s.available_memory
|
||||
|
||||
return memory_usage
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import os
|
||||
import types
|
||||
from importlib.metadata import version
|
||||
from importlib.util import find_spec
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -48,6 +49,17 @@ if HAS_TRITON:
|
||||
len(active_drivers),
|
||||
)
|
||||
HAS_TRITON = False
|
||||
|
||||
# Check Triton CPU
|
||||
if "cpu" in version("vllm"):
|
||||
if "cpu" in backends:
|
||||
HAS_TRITON = True
|
||||
else:
|
||||
logger.warning(
|
||||
"Triton is installed, but doesn't include CPU backend. "
|
||||
"Disabling Triton."
|
||||
)
|
||||
HAS_TRITON = False
|
||||
except ImportError:
|
||||
# This can occur if Triton is partially installed or triton.backends
|
||||
# is missing.
|
||||
|
||||
@@ -50,8 +50,10 @@ def is_pin_memory_available() -> bool:
|
||||
def is_uva_available() -> bool:
|
||||
"""Check if Unified Virtual Addressing (UVA) is available."""
|
||||
# UVA requires pinned memory.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# TODO: Add more requirements for UVA if needed.
|
||||
return is_pin_memory_available()
|
||||
return is_pin_memory_available() or current_platform.is_cpu()
|
||||
|
||||
|
||||
@cache
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
|
||||
|
||||
class UvaBuffer:
|
||||
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("UVA is not available")
|
||||
self.cpu = torch.zeros(size, dtype=dtype, device="cpu")
|
||||
self.np = self.cpu.numpy()
|
||||
self.uva = self.cpu
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CPUModelRunner(GPUModelRunner):
|
||||
# TBD: Whether need to move this to Worker?
|
||||
def warming_up_model(self) -> None:
|
||||
logger.info("Warming up model for the compilation...")
|
||||
# Only generate graph for the generic shape
|
||||
self.profile_run()
|
||||
logger.info("Warming up done.")
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# isort: skip_file
|
||||
# ruff: noqa: E402
|
||||
# mypy: disable-error-code="misc, assignment"
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Patch torch APIs
|
||||
import torch
|
||||
|
||||
|
||||
def noop(*args: Any, **kwargs: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _EventPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.record = noop
|
||||
self.synchronize = noop
|
||||
|
||||
|
||||
class _StreamPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.wait_stream = noop
|
||||
|
||||
def __enter__(self, *args, **kwargs):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
|
||||
torch.Event = _EventPlaceholder
|
||||
torch.cuda.Event = _EventPlaceholder
|
||||
torch.cuda.Stream = _StreamPlaceholder
|
||||
torch.cuda.set_stream = noop
|
||||
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
|
||||
torch.accelerator.synchronize = noop
|
||||
torch.accelerator.empty_cache = noop
|
||||
|
||||
# Patch vLLM torch utils
|
||||
import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
device: str | torch.device,
|
||||
pin_memory: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return torch.tensor(data, dtype=dtype, device="cpu")
|
||||
|
||||
|
||||
torch_utils.async_tensor_h2d = async_tensor_h2d
|
||||
|
||||
# Patch model runner APIs
|
||||
import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils
|
||||
import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils
|
||||
|
||||
gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer
|
||||
@@ -1,5 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Must be imported firstly
|
||||
import vllm.v1.worker.cpu.shm # noqa # isort: skip
|
||||
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
@@ -101,6 +105,8 @@ class CPUWorker(Worker):
|
||||
)
|
||||
|
||||
def init_device(self):
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
# Check whether critical libraries are loaded
|
||||
def check_preloaded_libs(name: str):
|
||||
ld_preload_list = os.environ.get("LD_PRELOAD", "")
|
||||
@@ -141,9 +147,16 @@ class CPUWorker(Worker):
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner: CPUModelRunner = CPUModelRunner(
|
||||
self.vllm_config, torch.device("cpu")
|
||||
)
|
||||
if self.use_v2_model_runner:
|
||||
from vllm.v1.worker.cpu.model_runner import (
|
||||
CPUModelRunner as CPUModelRunnerV2,
|
||||
)
|
||||
|
||||
self.model_runner: CPUModelRunner = CPUModelRunnerV2( # type: ignore
|
||||
self.vllm_config, self.device
|
||||
)
|
||||
else:
|
||||
self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
logger.warning("sleep mode is not supported on CPU, ignore it.")
|
||||
|
||||
Reference in New Issue
Block a user