[CPU] Experimentally enable Triton and MRV2 (#43225)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
|
||||
|
||||
class UvaBuffer:
|
||||
def __init__(self, size: int | Sequence[int], dtype: torch.dtype):
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("UVA is not available")
|
||||
self.cpu = torch.zeros(size, dtype=dtype, device="cpu")
|
||||
self.np = self.cpu.numpy()
|
||||
self.uva = self.cpu
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CPUModelRunner(GPUModelRunner):
|
||||
# TBD: Whether need to move this to Worker?
|
||||
def warming_up_model(self) -> None:
|
||||
logger.info("Warming up model for the compilation...")
|
||||
# Only generate graph for the generic shape
|
||||
self.profile_run()
|
||||
logger.info("Warming up done.")
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# isort: skip_file
|
||||
# ruff: noqa: E402
|
||||
# mypy: disable-error-code="misc, assignment"
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Patch torch APIs
|
||||
import torch
|
||||
|
||||
|
||||
def noop(*args: Any, **kwargs: Any) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _EventPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.record = noop
|
||||
self.synchronize = noop
|
||||
|
||||
|
||||
class _StreamPlaceholder:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.wait_stream = noop
|
||||
|
||||
def __enter__(self, *args, **kwargs):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
|
||||
torch.Event = _EventPlaceholder
|
||||
torch.cuda.Event = _EventPlaceholder
|
||||
torch.cuda.Stream = _StreamPlaceholder
|
||||
torch.cuda.set_stream = noop
|
||||
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
|
||||
torch.accelerator.synchronize = noop
|
||||
torch.accelerator.empty_cache = noop
|
||||
|
||||
# Patch vLLM torch utils
|
||||
import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
|
||||
def async_tensor_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
device: str | torch.device,
|
||||
pin_memory: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return torch.tensor(data, dtype=dtype, device="cpu")
|
||||
|
||||
|
||||
torch_utils.async_tensor_h2d = async_tensor_h2d
|
||||
|
||||
# Patch model runner APIs
|
||||
import vllm.v1.worker.gpu.buffer_utils as gpu_buffer_utils
|
||||
import vllm.v1.worker.cpu.buffer_utils as cpu_buffer_utils
|
||||
|
||||
gpu_buffer_utils.UvaBuffer = cpu_buffer_utils.UvaBuffer
|
||||
Reference in New Issue
Block a user