[CPU] Experimentally enable Triton and MRV2 (#43225)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
|
||||
|
||||
class UvaBuffer:
|
||||
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("UVA is not available")
|
||||
self.cpu = torch.zeros(size, dtype=dtype, device="cpu")
|
||||
self.np = self.cpu.numpy()
|
||||
self.uva = self.cpu
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CPUModelRunner(GPUModelRunner):
|
||||
# TBD: Whether need to move this to Worker?
|
||||
def warming_up_model(self) -> None:
|
||||
logger.info("Warming up model for the compilation...")
|
||||
# Only generate graph for the generic shape
|
||||
self.profile_run()
|
||||
logger.info("Warming up done.")
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# isort: skip_file
|
||||
# ruff: noqa: E402
|
||||
# mypy: disable-error-code="misc, assignment"
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Patch torch APIs
|
||||
import torch
|
||||
|
||||
|
||||
def noop(*args: Any, **kwargs: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _EventPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.record = noop
|
||||
self.synchronize = noop
|
||||
|
||||
|
||||
class _StreamPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.wait_stream = noop
|
||||
|
||||
def __enter__(self, *args, **kwargs):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
|
||||
torch.Event = _EventPlaceholder
|
||||
torch.cuda.Event = _EventPlaceholder
|
||||
torch.cuda.Stream = _StreamPlaceholder
|
||||
torch.cuda.set_stream = noop
|
||||
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
|
||||
torch.accelerator.synchronize = noop
|
||||
torch.accelerator.empty_cache = noop
|
||||
|
||||
# Patch vLLM torch utils
|
||||
import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
device: str | torch.device,
|
||||
pin_memory: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return torch.tensor(data, dtype=dtype, device="cpu")
|
||||
|
||||
|
||||
torch_utils.async_tensor_h2d = async_tensor_h2d
|
||||
|
||||
# Patch model runner APIs
|
||||
import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils
|
||||
import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils
|
||||
|
||||
gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer
|
||||
@@ -1,5 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Must be imported firstly
|
||||
import vllm.v1.worker.cpu.shm # noqa # isort: skip
|
||||
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
@@ -101,6 +105,8 @@ class CPUWorker(Worker):
|
||||
)
|
||||
|
||||
def init_device(self):
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
# Check whether critical libraries are loaded
|
||||
def check_preloaded_libs(name: str):
|
||||
ld_preload_list = os.environ.get("LD_PRELOAD", "")
|
||||
@@ -141,9 +147,16 @@ class CPUWorker(Worker):
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner: CPUModelRunner = CPUModelRunner(
|
||||
self.vllm_config, torch.device("cpu")
|
||||
)
|
||||
if self.use_v2_model_runner:
|
||||
from vllm.v1.worker.cpu.model_runner import (
|
||||
CPUModelRunner as CPUModelRunnerV2,
|
||||
)
|
||||
|
||||
self.model_runner: CPUModelRunner = CPUModelRunnerV2( # type: ignore
|
||||
self.vllm_config, self.device
|
||||
)
|
||||
else:
|
||||
self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
logger.warning("sleep mode is not supported on CPU, ignore it.")
|
||||
|
||||
Reference in New Issue
Block a user