[CPU] Experimentally enable Triton and MRV2 (#43225)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-05-22 01:48:17 -07:00
committed by GitHub
parent 2380bfc210
commit 65b7a812a2
11 changed files with 169 additions and 13 deletions
+20 -3
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.utils.cpu_resource_utils import (
DEVICE_CONTROL_ENV_VAR,
get_memory_node_info,
get_visible_memory_node,
)
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -135,9 +136,13 @@ class CpuPlatform(Platform):
scheduler_config.async_scheduling = False
parallel_config = vllm_config.parallel_config
# OMP requires the MP executor to function correctly, UniProc is not
# supported as it is not possible to set the OMP environment correctly
if parallel_config.distributed_executor_backend == "uni":
if (
os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING", "1") == "1"
and parallel_config.distributed_executor_backend == "uni"
):
# OMP requires the MP executor to function correctly, UniProc
# is not supported as it is not possible to set the OMP
# environment correctly
parallel_config.distributed_executor_backend = "mp"
if parallel_config.worker_cls == "auto":
@@ -481,3 +486,15 @@ class CpuPlatform(Platform):
slot_mapping,
isa,
)
@classmethod
def get_current_memory_usage(
cls, device: torch.types.Device | None = None
) -> float:
allowed_mem_node_list = get_visible_memory_node()
mem_status_list = [get_memory_node_info(i) for i in allowed_mem_node_list]
memory_usage = 0
for s in mem_status_list:
memory_usage += s.total_memory - s.available_memory
return memory_usage
+12
View File
@@ -3,6 +3,7 @@
import os
import types
from importlib.metadata import version
from importlib.util import find_spec
from vllm.logger import init_logger
@@ -48,6 +49,17 @@ if HAS_TRITON:
len(active_drivers),
)
HAS_TRITON = False
# Check Triton CPU
if "cpu" in version("vllm"):
if "cpu" in backends:
HAS_TRITON = True
else:
logger.warning(
"Triton is installed, but doesn't include CPU backend. "
"Disabling Triton."
)
HAS_TRITON = False
except ImportError:
# This can occur if Triton is partially installed or triton.backends
# is missing.
+3 -1
View File
@@ -50,8 +50,10 @@ def is_pin_memory_available() -> bool:
def is_uva_available() -> bool:
"""Check if Unified Virtual Addressing (UVA) is available."""
# UVA requires pinned memory.
from vllm.platforms import current_platform
# TODO: Add more requirements for UVA if needed.
return is_pin_memory_available()
return is_pin_memory_available() or current_platform.is_cpu()
@cache
View File
+16
View File
@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import torch
from vllm.utils.platform_utils import is_uva_available
class UvaBuffer:
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
if not is_uva_available():
raise RuntimeError("UVA is not available")
self.cpu = torch.zeros(size, dtype=dtype, device="cpu")
self.np = self.cpu.numpy()
self.uva = self.cpu
+16
View File
@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.logger import init_logger
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
logger = init_logger(__name__)
class CPUModelRunner(GPUModelRunner):
# TBD: Whether need to move this to Worker?
def warming_up_model(self) -> None:
logger.info("Warming up model for the compilation...")
# Only generate graph for the generic shape
self.profile_run()
logger.info("Warming up done.")
+62
View File
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# isort: skip_file
# ruff: noqa: E402
# mypy: disable-error-code="misc, assignment"
from typing import Any
# Patch torch APIs
import torch
def noop(*args: Any, **kwargs: Any) -> None:
pass
class _EventPlaceholder:
def __init__(self, *args, **kwargs) -> None:
self.record = noop
self.synchronize = noop
class _StreamPlaceholder:
def __init__(self, *args, **kwargs) -> None:
self.wait_stream = noop
def __enter__(self, *args, **kwargs):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
torch.Event = _EventPlaceholder
torch.cuda.Event = _EventPlaceholder
torch.cuda.Stream = _StreamPlaceholder
torch.cuda.set_stream = noop
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
torch.accelerator.synchronize = noop
torch.accelerator.empty_cache = noop
# Patch vLLM torch utils
import vllm.utils.torch_utils as torch_utils
def async_tensor_h2d(
data: list,
dtype: torch.dtype,
device: str | torch.device,
pin_memory: bool = False,
) -> torch.Tensor:
return torch.tensor(data, dtype=dtype, device="cpu")
torch_utils.async_tensor_h2d = async_tensor_h2d
# Patch model runner APIs
import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils
import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils
gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer
+16 -3
View File
@@ -1,5 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Must be imported firstly
import vllm.v1.worker.cpu.shm # noqa # isort: skip
import math
import os
import sys
@@ -101,6 +105,8 @@ class CPUWorker(Worker):
)
def init_device(self):
self.device = torch.device("cpu")
# Check whether critical libraries are loaded
def check_preloaded_libs(name: str):
ld_preload_list = os.environ.get("LD_PRELOAD", "")
@@ -141,9 +147,16 @@ class CPUWorker(Worker):
set_random_seed(self.model_config.seed)
# Construct the model runner
self.model_runner: CPUModelRunner = CPUModelRunner(
self.vllm_config, torch.device("cpu")
)
if self.use_v2_model_runner:
from vllm.v1.worker.cpu.model_runner import (
CPUModelRunner as CPUModelRunnerV2,
)
self.model_runner: CPUModelRunner = CPUModelRunnerV2( # type: ignore
self.vllm_config, self.device
)
else:
self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
def sleep(self, level: int = 1) -> None:
logger.warning("sleep mode is not supported on CPU, ignore it.")