[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -30,30 +30,6 @@ def vllm_version_matches_substr(substr: str) -> bool:
|
||||
return substr in vllm_version
|
||||
|
||||
|
||||
def tpu_platform_plugin() -> str | None:
|
||||
logger.debug("Checking if TPU platform is available.")
|
||||
|
||||
# Check for Pathways TPU proxy
|
||||
if envs.VLLM_TPU_USING_PATHWAYS:
|
||||
logger.debug("Confirmed TPU platform is available via Pathways proxy.")
|
||||
return "tpu_inference.platforms.tpu_platform.TpuPlatform"
|
||||
|
||||
# Check for libtpu installation
|
||||
try:
|
||||
# While it's technically possible to install libtpu on a
|
||||
# non-TPU machine, this is a very uncommon scenario. Therefore,
|
||||
# we assume that libtpu is installed only if the machine
|
||||
# has TPUs.
|
||||
|
||||
import libtpu # noqa: F401
|
||||
|
||||
logger.debug("Confirmed TPU platform is available.")
|
||||
return "vllm.platforms.tpu.TpuPlatform"
|
||||
except Exception as e:
|
||||
logger.debug("TPU platform is not available because: %s", str(e))
|
||||
return None
|
||||
|
||||
|
||||
def cuda_platform_plugin() -> str | None:
|
||||
is_cuda = False
|
||||
logger.debug("Checking if CUDA platform is available.")
|
||||
@@ -140,28 +116,6 @@ def rocm_platform_plugin() -> str | None:
|
||||
return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
|
||||
|
||||
|
||||
def xpu_platform_plugin() -> str | None:
|
||||
is_xpu = False
|
||||
logger.debug("Checking if XPU platform is available.")
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.distributed.is_xccl_available():
|
||||
dist_backend = "xccl"
|
||||
from vllm.platforms.xpu import XPUPlatform
|
||||
|
||||
XPUPlatform.dist_backend = dist_backend
|
||||
logger.debug("Confirmed %s backend is available.", XPUPlatform.dist_backend)
|
||||
|
||||
if hasattr(torch, "xpu") and torch.xpu.is_available():
|
||||
is_xpu = True
|
||||
logger.debug("Confirmed XPU platform is available.")
|
||||
except Exception as e:
|
||||
logger.debug("XPU platform is not available because: %s", str(e))
|
||||
|
||||
return "vllm.platforms.xpu.XPUPlatform" if is_xpu else None
|
||||
|
||||
|
||||
def _is_amd_zen_cpu() -> bool:
|
||||
"""Detect AMD CPU with AVX-512 via /proc/cpuinfo."""
|
||||
if not os.path.exists("/proc/cpuinfo"):
|
||||
@@ -229,10 +183,8 @@ def cpu_platform_plugin() -> str | None:
|
||||
|
||||
|
||||
builtin_platform_plugins = {
|
||||
"tpu": tpu_platform_plugin,
|
||||
"cuda": cuda_platform_plugin,
|
||||
"rocm": rocm_platform_plugin,
|
||||
"xpu": xpu_platform_plugin,
|
||||
"cpu": cpu_platform_plugin,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
from tpu_inference.platforms import (
|
||||
TpuPlatform as TpuInferencePlatform,
|
||||
)
|
||||
|
||||
TpuPlatform = TpuInferencePlatform # type: ignore
|
||||
USE_TPU_INFERENCE = True
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"tpu_inference not found, please install tpu_inference to run vllm on TPU"
|
||||
)
|
||||
pass
|
||||
@@ -1,619 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
# import custom ops, trigger op registration
|
||||
import vllm_xpu_kernels._C # noqa
|
||||
import vllm_xpu_kernels._moe_C # noqa
|
||||
import vllm_xpu_kernels._xpu_C # noqa
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import DeviceCapability, Platform, PlatformEnum
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
VllmConfig = None
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_mem_info_wrapper(
|
||||
device: int | str | torch.device | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Get memory info for a device, matching `torch.accelerator.get_memory_info`.
|
||||
|
||||
Args:
|
||||
device: Device specification. Can be:
|
||||
- None: Use current XPU device
|
||||
- int: Device index
|
||||
- str: Device string (e.g., "xpu:0", "xpu")
|
||||
- torch.device: Device object
|
||||
|
||||
Returns:
|
||||
Tuple[int, int]: (free_memory, total_memory) in bytes
|
||||
|
||||
"""
|
||||
# Handle None - use current device
|
||||
if device is None:
|
||||
device = torch.xpu.current_device()
|
||||
|
||||
# Handle torch.device objects
|
||||
elif isinstance(device, torch.device):
|
||||
if device.type != "xpu":
|
||||
raise RuntimeError(f"Expected 'xpu' device, got '{device.type}'")
|
||||
# If device index is not specified, use current device
|
||||
device = (
|
||||
device.index if device.index is not None else torch.xpu.current_device()
|
||||
)
|
||||
|
||||
# Handle string device specifications (e.g., "xpu:0", "xpu")
|
||||
elif isinstance(device, str):
|
||||
if not device.startswith("xpu"):
|
||||
raise RuntimeError(f"Expected 'xpu' device string, got '{device}'")
|
||||
# Parse device string
|
||||
parts = device.split(":")
|
||||
if len(parts) == 1:
|
||||
# "xpu" -> use current device
|
||||
device = torch.xpu.current_device()
|
||||
elif len(parts) == 2:
|
||||
# "xpu:0" -> use index 0
|
||||
try:
|
||||
device = int(parts[1])
|
||||
except ValueError as err:
|
||||
raise RuntimeError(
|
||||
f"Invalid device index: '{device}', expected integer after ':'"
|
||||
) from err
|
||||
else:
|
||||
raise RuntimeError(f"Invalid device string format: '{device}'")
|
||||
|
||||
# At this point, device should be an int
|
||||
if isinstance(device, int):
|
||||
# bounds check
|
||||
device_count = torch.xpu.device_count()
|
||||
if not (0 <= device < device_count):
|
||||
raise ValueError(
|
||||
f"Invalid device index {device}, must be in range [0, {device_count})"
|
||||
)
|
||||
|
||||
elif not isinstance(device, int):
|
||||
raise TypeError(
|
||||
f"device must be int, str, torch.device, or None, got {type(device)}"
|
||||
)
|
||||
|
||||
# Call the underlying C++ implementation
|
||||
free, total = torch.ops._C_cache_ops.getMemoryInfo(device)
|
||||
|
||||
return free, total
|
||||
|
||||
|
||||
torch.accelerator.get_memory_info = get_mem_info_wrapper
|
||||
|
||||
|
||||
class XPUPlatform(Platform):
|
||||
_enum = PlatformEnum.XPU
|
||||
device_name: str = "xpu"
|
||||
device_type: str = "xpu"
|
||||
dispatch_key: str = "XPU"
|
||||
# Intel XPU's device key is "GPU" for Ray.
|
||||
# see https://github.com/ray-project/ray/blob/6a5eb5865eeb9ccf058a79b44f107e327e360673/python/ray/_private/accelerators/intel_gpu.py#L20 # noqa: E501
|
||||
ray_device_key: str = "GPU"
|
||||
dist_backend: str = "xccl" # xccl only
|
||||
device_control_env_var: str = "ZE_AFFINITY_MASK"
|
||||
supported_quantization: list[str] = [
|
||||
"awq",
|
||||
"gptq",
|
||||
"moe_wna16",
|
||||
"auto_awq",
|
||||
"auto_gptq",
|
||||
"inc",
|
||||
"fp8",
|
||||
"deepseek_v4_fp8",
|
||||
"mxfp4",
|
||||
"mxfp8",
|
||||
"fp8_per_tensor",
|
||||
"fp8_per_block",
|
||||
"fp8_per_channel",
|
||||
"online",
|
||||
"gpt_oss_mxfp4",
|
||||
"modelopt",
|
||||
"compressed-tensors",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def import_kernels(cls) -> None:
|
||||
# Do not import vllm._C
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._moe_C # noqa: F401
|
||||
|
||||
@classmethod
|
||||
def check_runner_kv_caches_multi_layer(cls) -> None:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
selected_backend: "AttentionBackendEnum",
|
||||
attn_selector_config: "AttentionSelectorConfig",
|
||||
num_heads: int | None = None,
|
||||
) -> str:
|
||||
# TurboQuant KV cache: route directly to TQ backend
|
||||
kv_cache_dtype = attn_selector_config.kv_cache_dtype
|
||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
|
||||
logger.info_once("Using TurboQuant attention backend.")
|
||||
return AttentionBackendEnum.TURBOQUANT.get_path()
|
||||
|
||||
dtype = attn_selector_config.dtype
|
||||
if attn_selector_config.use_sparse:
|
||||
logger.info_once("Using XPU MLA Sparse backend.")
|
||||
return AttentionBackendEnum.XPU_MLA_SPARSE.get_path()
|
||||
if attn_selector_config.use_mla:
|
||||
if selected_backend == AttentionBackendEnum.TRITON_MLA:
|
||||
logger.info_once("Using Triton MLA backend on V1 engine.")
|
||||
return AttentionBackendEnum.TRITON_MLA.get_path()
|
||||
if selected_backend not in (None, AttentionBackendEnum.FLASH_ATTN_MLA):
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for {cls.device_name}, "
|
||||
f"with use_mla: {attn_selector_config.use_mla}"
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashattn_mla import (
|
||||
XPUFlashAttnMLABackend,
|
||||
)
|
||||
|
||||
invalid_reasons = XPUFlashAttnMLABackend.validate_configuration(
|
||||
device_capability=cls.get_device_capability(), # type: ignore[arg-type]
|
||||
**attn_selector_config._asdict(),
|
||||
)
|
||||
if invalid_reasons:
|
||||
if selected_backend == AttentionBackendEnum.FLASH_ATTN_MLA:
|
||||
raise ValueError(
|
||||
f"FlashAttnMLA on XPU is not valid for this "
|
||||
f"configuration: {invalid_reasons}"
|
||||
)
|
||||
logger.info_once(
|
||||
"FlashAttnMLA on XPU is not valid for this configuration "
|
||||
"(%s); falling back to Triton MLA backend.",
|
||||
tuple(invalid_reasons),
|
||||
)
|
||||
return AttentionBackendEnum.TRITON_MLA.get_path()
|
||||
logger.info_once("Using Flash Attention MLA backend on XPU.")
|
||||
return "vllm.v1.attention.backends.mla.flashattn_mla.XPUFlashAttnMLABackend"
|
||||
if selected_backend == AttentionBackendEnum.TRITON_ATTN:
|
||||
logger.info_once("Using Triton backend.")
|
||||
return AttentionBackendEnum.TRITON_ATTN.get_path()
|
||||
elif attn_selector_config.use_batch_invariant:
|
||||
# Flash Attention on XPU has not been validated for batch
|
||||
# invariance. Honor an explicit Flash Attention request;
|
||||
# otherwise fall back to Triton Attention, which implements
|
||||
# batch-invariant kernels.
|
||||
if selected_backend == AttentionBackendEnum.FLASH_ATTN:
|
||||
logger.warning_once(
|
||||
"Using Flash Attention on XPU with batch invariance "
|
||||
"enabled because it was explicitly requested. This "
|
||||
"backend has not been validated for batch invariance "
|
||||
"on XPU and may produce non-deterministic results "
|
||||
"across batch sizes."
|
||||
)
|
||||
return AttentionBackendEnum.FLASH_ATTN.get_path()
|
||||
logger.info_once(
|
||||
"VLLM_BATCH_INVARIANT is enabled. Using Triton Attention "
|
||||
"backend on XPU, which implements batch-invariant kernels."
|
||||
)
|
||||
return AttentionBackendEnum.TRITON_ATTN.get_path()
|
||||
elif attn_selector_config.use_mm_prefix:
|
||||
# Flash Attention on XPU has no FA4 kernel, so it cannot apply the
|
||||
# multimodal prefix-LM bidirectional mask. Honor an explicit Flash
|
||||
# Attention request (for text-only workloads); otherwise fall back
|
||||
# to Triton Attention, which supports mm_prefix.
|
||||
if selected_backend == AttentionBackendEnum.FLASH_ATTN:
|
||||
logger.warning_once(
|
||||
"Using Flash Attention on XPU for a multimodal prefix-LM "
|
||||
"model because it was explicitly requested. The prefix-LM "
|
||||
"bidirectional mask cannot be applied, so image/video "
|
||||
"inputs will produce incorrect results; only use this for "
|
||||
"text-only workloads."
|
||||
)
|
||||
return AttentionBackendEnum.FLASH_ATTN.get_path()
|
||||
logger.warning_once(
|
||||
"Flash Attention on XPU does not support multimodal prefix-LM "
|
||||
"attention. Falling back to Triton Attention backend."
|
||||
)
|
||||
return AttentionBackendEnum.TRITON_ATTN.get_path()
|
||||
elif dtype == torch.float32:
|
||||
logger.warning_once(
|
||||
"Flash Attention on XPU does not support float32 dtype. "
|
||||
"Falling back to Triton Attention backend."
|
||||
)
|
||||
return AttentionBackendEnum.TRITON_ATTN.get_path()
|
||||
elif selected_backend == AttentionBackendEnum.FLASH_ATTN:
|
||||
logger.info_once("Using Flash Attention backend.")
|
||||
return AttentionBackendEnum.FLASH_ATTN.get_path()
|
||||
elif selected_backend:
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for {cls.device_name}, "
|
||||
f"with use_mla: {attn_selector_config.use_mla}"
|
||||
)
|
||||
|
||||
logger.info_once("Using Flash Attention backend.")
|
||||
return AttentionBackendEnum.FLASH_ATTN.get_path()
|
||||
|
||||
@classmethod
|
||||
def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]:
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_vit_attn_backend(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
backend: "AttentionBackendEnum | None" = None,
|
||||
) -> "AttentionBackendEnum":
|
||||
if dtype == torch.float32:
|
||||
logger.warning_once(
|
||||
"Flash Attention on XPU does not support float32 dtype. "
|
||||
"Falling back to Triton Attention backend for vit attention."
|
||||
)
|
||||
return AttentionBackendEnum.TRITON_ATTN
|
||||
|
||||
if backend is not None:
|
||||
assert backend in cls.get_supported_vit_attn_backends(), (
|
||||
f"Backend {backend} is not supported for vit attention. "
|
||||
f"Supported backends are: "
|
||||
f"{cls.get_supported_vit_attn_backends()}."
|
||||
)
|
||||
logger.info_once(f"Using backend {backend} for vit attention")
|
||||
return backend
|
||||
|
||||
logger.info_once(
|
||||
f"Using backend {AttentionBackendEnum.FLASH_ATTN} for vit attention"
|
||||
)
|
||||
return AttentionBackendEnum.FLASH_ATTN
|
||||
|
||||
@classmethod
|
||||
def set_device(cls, device: torch.device) -> None:
|
||||
"""Set the device for the current platform."""
|
||||
torch.xpu.set_device(device)
|
||||
|
||||
@classmethod
|
||||
def manual_seed_all(cls, seed: int) -> None:
|
||||
torch.xpu.manual_seed_all(seed)
|
||||
|
||||
@classmethod
|
||||
def get_device_capability(
|
||||
cls,
|
||||
device_id: int = 0,
|
||||
) -> DeviceCapability | None:
|
||||
# capacity format differs from cuda's and will cause unexpected
|
||||
# failure, so use None directly
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_device_name(cls, device_id: int = 0) -> str:
|
||||
return torch.xpu.get_device_name(device_id)
|
||||
|
||||
@classmethod
|
||||
def get_punica_wrapper(cls) -> str:
|
||||
xpu_use_triton_kernel = os.getenv("XPU_USE_TRITON_KERNEL", "0") == "1"
|
||||
if not xpu_use_triton_kernel:
|
||||
return "vllm.lora.punica_wrapper.punica_xpu.PunicaWrapperXPU"
|
||||
else:
|
||||
return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"
|
||||
|
||||
@classmethod
|
||||
def get_device_total_memory(cls, device_id: int = 0) -> int:
|
||||
device_props = torch.xpu.get_device_properties(device_id)
|
||||
return device_props.total_memory
|
||||
|
||||
@classmethod
|
||||
def inference_mode(cls):
|
||||
return torch.no_grad()
|
||||
|
||||
@classmethod
|
||||
def get_static_graph_wrapper_cls(cls) -> str:
|
||||
return "vllm.compilation.cuda_graph.CUDAGraphWrapper"
|
||||
|
||||
@classmethod
|
||||
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
|
||||
# lazy import to avoid circular import
|
||||
from vllm.config import CUDAGraphMode
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
model_config = vllm_config.model_config
|
||||
if model_config is not None and (
|
||||
model_config.quantization is not None
|
||||
or vllm_config.quant_config is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"XPU batch invariance currently supports only unquantized "
|
||||
f"models; got quantization={model_config.quantization!r}. "
|
||||
"Use an unquantized model or disable VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
|
||||
cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
if cache_dtype not in ("auto", "float16", "bfloat16"):
|
||||
raise ValueError(
|
||||
"XPU batch invariance currently does not support quantized "
|
||||
f"KV caches; got kv_cache_dtype={cache_dtype!r}. "
|
||||
"Use an unquantized KV cache dtype or disable "
|
||||
"VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if compilation_config.compile_sizes is None:
|
||||
compilation_config.compile_sizes = []
|
||||
|
||||
# lazy import to avoid circular import
|
||||
from vllm.utils.torch_utils import supports_xpu_graph
|
||||
|
||||
if not supports_xpu_graph():
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
logger.warning_once(
|
||||
"XPU Graph is not supported in the current PyTorch version, "
|
||||
"disabling cudagraph_mode."
|
||||
)
|
||||
|
||||
if (
|
||||
vllm_config.model_config is not None
|
||||
and vllm_config.model_config.enable_sleep_mode
|
||||
and compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
):
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
logger.warning_once(
|
||||
"XPU Graph is not compatible with sleep mode, disabling cudagraph_mode."
|
||||
)
|
||||
|
||||
# Disable fusion passes not yet supported on XPU.
|
||||
from vllm.config.compilation import CompilationMode
|
||||
|
||||
pass_config = compilation_config.pass_config
|
||||
fusion_passes_to_disable = {
|
||||
"fuse_allreduce_rms": "AllReduce + RMSNorm fusion",
|
||||
"fuse_attn_quant": "Attention + quant fusion",
|
||||
"fuse_act_padding": "Activation + padding fusion",
|
||||
"fuse_rope_kvcache": "RoPE + KV cache fusion",
|
||||
"fuse_rope_kvcache_cat_mla": "RoPE + KV cache + MLA fusion",
|
||||
}
|
||||
if compilation_config.mode != CompilationMode.NONE:
|
||||
for flag, feature_name in fusion_passes_to_disable.items():
|
||||
if getattr(pass_config, flag):
|
||||
logger.warning_once(
|
||||
"Feature %r is not yet supported on XPU and will be disabled.",
|
||||
feature_name,
|
||||
)
|
||||
setattr(pass_config, flag, False)
|
||||
|
||||
# UVA-offloaded weights are host USM allocations, which Inductor's
|
||||
# static Triton launcher rejects ("Pointer argument doesn't reference
|
||||
# XPU device memory"). Fall back to Triton's own launcher. Remove once
|
||||
# the released torch contains pytorch/pytorch#188240, which relaxes
|
||||
# that check to any memory type known by the driver.
|
||||
offload_config = vllm_config.offload_config
|
||||
uva_offloading = offload_config.offload_backend == "uva" or (
|
||||
offload_config.offload_backend == "auto"
|
||||
and offload_config.prefetch.offload_group_size == 0
|
||||
and offload_config.uva.cpu_offload_gb > 0
|
||||
)
|
||||
if (
|
||||
uva_offloading
|
||||
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_UVA
|
||||
and compilation_config.mode != CompilationMode.NONE
|
||||
):
|
||||
compilation_config.inductor_compile_config.setdefault(
|
||||
"use_static_cuda_launcher", False
|
||||
)
|
||||
logger.info_once(
|
||||
"Disabling Inductor's static Triton launcher because UVA "
|
||||
"weight offloading is enabled."
|
||||
)
|
||||
|
||||
# check and update parallel config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# Only override worker_cls if it's still the default "auto"
|
||||
# This allows custom workers (like vllm-omni workers) to be used on XPU
|
||||
if parallel_config.worker_cls == "auto":
|
||||
parallel_config.worker_cls = "vllm.v1.worker.xpu_worker.XPUWorker"
|
||||
if vllm_config.kv_transfer_config is not None:
|
||||
vllm_config.kv_transfer_config.enable_permute_local_kv = True
|
||||
|
||||
# In some cases, the internal memory type cache can misdetect GPU
|
||||
# memory as host memory, also leading to invalid memory access.
|
||||
# This cache can be disabled by setting UCX_MEMTYPE_CACHE=n.
|
||||
# ref. https://openucx.readthedocs.io/en/master/faq.html
|
||||
os.environ["UCX_MEMTYPE_CACHE"] = "n"
|
||||
|
||||
# spawn is the only supported multiprocessing method on XPU
|
||||
if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
# XPU requires graceful shutdown to allow oneCCL/Level Zero resources
|
||||
# to be properly released. Without this, subsequent server startups on
|
||||
# the same devices may hang during CCL initialization.
|
||||
if vllm_config.shutdown_timeout == 0:
|
||||
vllm_config.shutdown_timeout = 5
|
||||
logger.info(
|
||||
"XPU platform: set server shutdown_timeout=%d.",
|
||||
vllm_config.shutdown_timeout,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
super().update_block_size_for_backend(vllm_config)
|
||||
from vllm.config.vllm import get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import (
|
||||
AttentionLayerBase,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
# special fix for GDN since kernel only supports block size dividable by 64
|
||||
attn_layers = get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
|
||||
kernel_block_size = None
|
||||
for layer in attn_layers.values():
|
||||
b = layer.get_attn_backend()
|
||||
if b.get_name() == "GDN_ATTN":
|
||||
kernel_block_size = 64
|
||||
break
|
||||
|
||||
if kernel_block_size is None:
|
||||
return
|
||||
new_block_size = (
|
||||
cdiv(cache_config.block_size, kernel_block_size) * kernel_block_size
|
||||
)
|
||||
if new_block_size == cache_config.block_size:
|
||||
return
|
||||
|
||||
pre_block_size = cache_config.block_size
|
||||
if cache_config.mamba_cache_mode == "align":
|
||||
cache_config.mamba_block_size = new_block_size
|
||||
original_mamba_page_size_padded = cache_config.mamba_page_size_padded
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
attn_page_size_1_token = (
|
||||
cache_config.mamba_page_size_padded // cache_config.block_size
|
||||
)
|
||||
cache_config.mamba_page_size_padded = (
|
||||
new_block_size * attn_page_size_1_token
|
||||
)
|
||||
cache_config.block_size = new_block_size
|
||||
logger.info(
|
||||
"[XPU]Setting attention block size to %d tokens to ensure multiple of %d.",
|
||||
new_block_size,
|
||||
kernel_block_size,
|
||||
)
|
||||
if original_mamba_page_size_padded is not None:
|
||||
logger.info(
|
||||
"[XPU]Scaled mamba_page_size_padded from %d to %d bytes accordingly.",
|
||||
original_mamba_page_size_padded,
|
||||
cache_config.mamba_page_size_padded,
|
||||
)
|
||||
# This rounding runs after super()'s check, so check its result too.
|
||||
cls._check_aligned_block_size(
|
||||
vllm_config, cls._find_non_ssm_backends(vllm_config), pre_block_size
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def support_hybrid_kv_cache(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def support_static_graph_mode(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def is_pin_memory_available(cls):
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_current_memory_usage(
|
||||
cls, device: torch.types.Device | None = None
|
||||
) -> float:
|
||||
torch.xpu.empty_cache()
|
||||
torch.xpu.reset_peak_memory_stats(device)
|
||||
return torch.xpu.max_memory_allocated(device)
|
||||
|
||||
@classmethod
|
||||
def fp8_dtype(cls) -> torch.dtype:
|
||||
return torch.float8_e4m3fn
|
||||
|
||||
@classmethod
|
||||
def is_data_center_gpu(cls) -> bool:
|
||||
device_name = cls.get_device_name().lower()
|
||||
return device_name.count("data center gpu") > 0
|
||||
|
||||
@classmethod
|
||||
def get_device_communicator_cls(cls) -> str:
|
||||
if not torch.distributed.is_xccl_available():
|
||||
# Supports xccl with PyTorch versions >= 2.8.0.dev for XPU platform
|
||||
logger.warning(
|
||||
"xccl is not enabled in this torch build, communication"
|
||||
" is not available."
|
||||
)
|
||||
return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa
|
||||
|
||||
@classmethod
|
||||
def supports_fp8(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_default_ir_op_priority(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> "IrOpPriorityConfig":
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
# Native used by default when compiling,
|
||||
# use fused kernels where available when no codegen
|
||||
cc = vllm_config.compilation_config
|
||||
using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE
|
||||
default = ["native"] if using_inductor else ["vllm_c", "native"]
|
||||
|
||||
return IrOpPriorityConfig.with_default(default, gelu_and_mul_sparse=["native"])
|
||||
|
||||
@classmethod
|
||||
def device_count(cls) -> int:
|
||||
return torch.xpu.device_count()
|
||||
|
||||
@classmethod
|
||||
def check_if_supports_dtype(cls, dtype: torch.dtype):
|
||||
if dtype == torch.bfloat16: # noqa: SIM102
|
||||
device_name = cls.get_device_name().lower()
|
||||
# client gpu a770
|
||||
if device_name.count("a770") > 0:
|
||||
raise ValueError(
|
||||
"Intel Arc A770 have bfloat16 accuracy known issue. "
|
||||
"You can use float16 instead by explicitly setting the "
|
||||
"`dtype` flag in CLI, for example: --dtype=half."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def opaque_attention_op(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def insert_blocks_to_device(
|
||||
cls,
|
||||
src_cache: torch.Tensor,
|
||||
dst_cache: torch.Tensor,
|
||||
src_block_indices: torch.Tensor,
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from src_cache to dst_cache on XPU."""
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.to(dst_cache.device)
|
||||
|
||||
@classmethod
|
||||
def swap_out_blocks_to_host(
|
||||
cls,
|
||||
src_cache: torch.Tensor,
|
||||
dst_cache: torch.Tensor,
|
||||
src_block_indices: torch.Tensor,
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy blocks from XPU to host (CPU)."""
|
||||
_src_cache = src_cache[src_block_indices]
|
||||
dst_cache[dst_block_indices] = _src_cache.cpu()
|
||||
|
||||
@classmethod
|
||||
def num_compute_units(cls, device_id: int = 0) -> int:
|
||||
return torch.xpu.get_device_properties(device_id).max_compute_units
|
||||
|
||||
@classmethod
|
||||
def use_custom_op_collectives(cls) -> bool:
|
||||
return True
|
||||
Reference in New Issue
Block a user