replace cuda_device_count_stateless() to current_platform.device_count() (#37841)

Signed-off-by: Liao, Wei <wei.liao@intel.com>
Signed-off-by: wliao2 <wei.liao@intel.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
wliao2
2026-03-31 22:32:54 +08:00
committed by GitHub
co-authored by Harry Mellor Kunshang Ji
parent e8057c00bc
commit 4dfad17ed1
20 changed files with 96 additions and 92 deletions
+29 -3
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
import os
from collections.abc import Callable
from datetime import timedelta
from functools import cache, wraps
from functools import cache, lru_cache, wraps
from typing import TYPE_CHECKING, TypeVar
import torch
@@ -20,9 +20,9 @@ from typing_extensions import ParamSpec
# import custom ops, trigger op registration
import vllm._C # noqa
import vllm._C_stable_libtorch # noqa
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.import_utils import import_pynvml
from vllm.utils.torch_utils import cuda_device_count_stateless
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interface import DeviceCapability, Platform, PlatformEnum
@@ -47,6 +47,32 @@ pynvml = import_pynvml()
torch.backends.cuda.enable_cudnn_sdp(False)
@lru_cache(maxsize=8)
def _cuda_device_count_stateless(cuda_visible_devices: str | None = None) -> int:
"""Get number of CUDA devices, caching based on the value of CUDA_VISIBLE_DEVICES
at the time of call.
This should be used instead of torch.accelerator.device_count() unless
CUDA_VISIBLE_DEVICES has already been set to the desired value.
# This can be removed and simply replaced with torch.cuda.get_device_count
# after https://github.com/pytorch/pytorch/pull/122815 is released."""
# Note: cuda_visible_devices is not used, but we keep it as an argument for
# LRU Cache purposes.
# Code below is based on
# https://github.com/pytorch/pytorch/blob/
# c1cd946818442aca8c7f812b16d187ce1586c3bc/
# torch/cuda/__init__.py#L831C1-L831C17
import torch.cuda
if not torch.cuda._is_compiled():
return 0
raw_count = torch.cuda._device_count_nvml()
r = torch._C._cuda_getDeviceCount() if raw_count < 0 else raw_count
return r
@cache
def _get_backend_priorities(
use_mla: bool,
@@ -456,7 +482,7 @@ class CudaPlatformBase(Platform):
@classmethod
def device_count(cls) -> int:
return cuda_device_count_stateless()
return _cuda_device_count_stateless(envs.CUDA_VISIBLE_DEVICES)
@classmethod
def check_if_supports_dtype(cls, dtype: torch.dtype):
+33 -2
View File
@@ -13,7 +13,6 @@ from torch.distributed.distributed_c10d import is_nccl_available
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.torch_utils import cuda_device_count_stateless
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interface import DeviceCapability, Platform, PlatformEnum
@@ -67,6 +66,38 @@ _ROCM_DEVICE_ID_NAME_MAP: dict[str, str] = {
}
@lru_cache(maxsize=8)
def _rocm_device_count_stateless(cuda_visible_devices: str | None = None) -> int:
"""Get number of ROCm devices, caching based on the value of CUDA_VISIBLE_DEVICES
at the time of call.
This should be used instead of torch.accelerator.device_count() unless
CUDA_VISIBLE_DEVICES has already been set to the desired value.
# This can be removed and simply replaced with torch.cuda.get_device_count
# after https://github.com/pytorch/pytorch/pull/122815 is released."""
# Note: cuda_visible_devices is not used, but we keep it as an argument for
# LRU Cache purposes.
# Code below is based on
# https://github.com/pytorch/pytorch/blob/
# c1cd946818442aca8c7f812b16d187ce1586c3bc/
# torch/cuda/__init__.py#L831C1-L831C17
import torch.cuda
if not torch.cuda._is_compiled():
return 0
# ROCm uses amdsmi instead of nvml for stateless device count
# This requires a sufficiently modern version of Torch 2.4.0
raw_count = (
torch.cuda._device_count_amdsmi()
if (hasattr(torch.cuda, "_device_count_amdsmi"))
else -1
)
r = torch._C._cuda_getDeviceCount() if raw_count < 0 else raw_count
return r
def _sync_hip_cuda_env_vars():
"""Ensure HIP_VISIBLE_DEVICES and CUDA_VISIBLE_DEVICES are consistent.
Treats empty string as unset. Raises on genuine conflicts."""
@@ -810,7 +841,7 @@ class RocmPlatform(Platform):
@classmethod
def device_count(cls) -> int:
return cuda_device_count_stateless()
return _rocm_device_count_stateless(getattr(envs, cls.device_control_env_var))
@classmethod
def check_if_supports_dtype(cls, dtype: torch.dtype):