Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
135 lines
4.4 KiB
Python
135 lines
4.4 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import logging
|
|
import traceback
|
|
from typing import TYPE_CHECKING
|
|
|
|
from vllm.utils.import_utils import resolve_obj_by_qualname
|
|
|
|
from .interface import CpuArchEnum, Platform, PlatformEnum
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def vllm_version_matches_substr(substr: str) -> bool:
|
|
"""Check to see if the vLLM version matches a substring."""
|
|
from importlib.metadata import PackageNotFoundError, version
|
|
|
|
try:
|
|
vllm_version = version("vllm")
|
|
except PackageNotFoundError as e:
|
|
logger.warning(
|
|
"The vLLM package was not found, so its version could not be "
|
|
"inspected. This may cause platform detection to fail."
|
|
)
|
|
raise e
|
|
return substr in vllm_version
|
|
|
|
|
|
def cuda_platform_plugin() -> str | None:
|
|
is_cuda = False
|
|
logger.debug("Checking if CUDA platform is available.")
|
|
try:
|
|
from vllm.utils.import_utils import import_pynvml
|
|
|
|
pynvml = import_pynvml()
|
|
pynvml.nvmlInit()
|
|
try:
|
|
# NOTE: Edge case: vllm cpu build on a GPU machine.
|
|
# Third-party pynvml can be imported in cpu build,
|
|
# we need to check if vllm is built with cpu too.
|
|
# Otherwise, vllm will always activate cuda plugin
|
|
# on a GPU machine, even if in a cpu build.
|
|
is_cuda = (
|
|
pynvml.nvmlDeviceGetCount() > 0
|
|
and not vllm_version_matches_substr("cpu")
|
|
)
|
|
if pynvml.nvmlDeviceGetCount() <= 0:
|
|
logger.debug("CUDA platform is not available because no GPU is found.")
|
|
if vllm_version_matches_substr("cpu"):
|
|
logger.debug(
|
|
"CUDA platform is not available because vLLM is built with CPU."
|
|
)
|
|
if is_cuda:
|
|
logger.debug("Confirmed CUDA platform is available.")
|
|
finally:
|
|
pynvml.nvmlShutdown()
|
|
except Exception as e:
|
|
logger.debug("Exception happens when checking CUDA platform: %s", str(e))
|
|
if "nvml" not in e.__class__.__name__.lower():
|
|
# If the error is not related to NVML, re-raise it.
|
|
raise e
|
|
|
|
# CUDA is supported on Jetson, but NVML may not be.
|
|
import os
|
|
|
|
def cuda_is_jetson() -> bool:
|
|
return os.path.isfile("/etc/nv_tegra_release") or os.path.exists(
|
|
"/sys/class/tegra-firmware"
|
|
)
|
|
|
|
if cuda_is_jetson():
|
|
logger.debug("Confirmed CUDA platform is available on Jetson.")
|
|
is_cuda = True
|
|
else:
|
|
logger.debug("CUDA platform is not available because: %s", str(e))
|
|
|
|
return "vllm.platforms.cuda.CudaPlatform" if is_cuda else None
|
|
|
|
|
|
def resolve_current_platform_cls_qualname() -> str:
|
|
try:
|
|
platform_cls_qualname = cuda_platform_plugin()
|
|
except Exception:
|
|
logger.debug("Platform plugin cuda failed during detection.", exc_info=True)
|
|
platform_cls_qualname = None
|
|
if platform_cls_qualname is not None:
|
|
logger.debug("Automatically detected platform cuda.")
|
|
return platform_cls_qualname
|
|
logger.debug(
|
|
"No platform detected, vLLM is running on UnspecifiedPlatform",
|
|
)
|
|
return "vllm.platforms.interface.UnspecifiedPlatform"
|
|
|
|
|
|
_current_platform = None
|
|
_init_trace: str = ""
|
|
|
|
if TYPE_CHECKING:
|
|
current_platform: Platform
|
|
|
|
|
|
def __getattr__(name: str):
|
|
if name == "current_platform":
|
|
# lazy init current_platform.
|
|
global _current_platform
|
|
if _current_platform is None:
|
|
platform_cls_qualname = resolve_current_platform_cls_qualname()
|
|
_current_platform = resolve_obj_by_qualname(platform_cls_qualname)()
|
|
global _init_trace
|
|
_init_trace = "".join(traceback.format_stack())
|
|
return _current_platform
|
|
elif name in globals():
|
|
return globals()[name]
|
|
else:
|
|
raise AttributeError(f"No attribute named '{name}' exists in {__name__}.")
|
|
|
|
|
|
def __setattr__(name: str, value):
|
|
if name == "current_platform":
|
|
global _current_platform
|
|
_current_platform = value
|
|
elif name in globals():
|
|
globals()[name] = value
|
|
else:
|
|
raise AttributeError(f"No attribute named '{name}' exists in {__name__}.")
|
|
|
|
|
|
__all__ = [
|
|
"Platform",
|
|
"PlatformEnum",
|
|
"current_platform",
|
|
"CpuArchEnum",
|
|
"_init_trace",
|
|
]
|