# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import os from functools import wraps from typing import TYPE_CHECKING import torch import vllm.envs as envs from vllm.logger import init_logger if TYPE_CHECKING: pass logger = init_logger(__name__) try: from amdsmi import ( amdsmi_get_gpu_asic_info, amdsmi_get_processor_handles, amdsmi_init, amdsmi_shut_down, ) except ImportError as e: logger.warning("Failed to import from amdsmi with %r", e) try: import vllm._C # noqa: F401 except ImportError as e: logger.warning("Failed to import from vllm._C with %r", e) # import custom ops, trigger op registration try: import vllm._C_stable_libtorch # noqa: F401 except ImportError as e: logger.warning("Failed to import from vllm._C_stable_libtorch with %r", e) # Models not supported by ROCm. _ROCM_UNSUPPORTED_MODELS: list[str] = [] # Models partially supported by ROCm. # Architecture -> Reason. _ROCM_PARTIALLY_SUPPORTED_MODELS: dict[str, str] = {} _ROCM_DEVICE_ID_NAME_MAP: dict[str, str] = { "0x74a0": "AMD_Instinct_MI300A", "0x74a1": "AMD_Instinct_MI300X", "0x74b5": "AMD_Instinct_MI300X", # MI300X VF "0x74a2": "AMD_Instinct_MI308X", "0x74a5": "AMD_Instinct_MI325X", "0x74b9": "AMD_Instinct_MI325X", # MI325X VF "0x74a9": "AMD_Instinct_MI300X_HF", "0x74bd": "AMD_Instinct_MI300X_HF", "0x744c": "AMD_Radeon_RX7900XTX", # RDNA 3.5 APUs (Strix Point / Strix Halo) "0x150e": "AMD_Radeon_890M", # gfx1150, Strix Point "0x1586": "AMD_Radeon_8060S", # gfx1151, Strix Halo # RDNA 4 discrete (Navi 48) "0x7550": "AMD_Radeon_RX9070XT", # gfx1201 "0x7551": "AMD_Radeon_R9700", # gfx1201 } def _sync_hip_cuda_env_vars(): """Ensure HIP_VISIBLE_DEVICES and CUDA_VISIBLE_DEVICES are consistent. Treats empty string as unset. Raises on genuine conflicts.""" hip_val = os.environ.get("HIP_VISIBLE_DEVICES") or None cuda_val = os.environ.get("CUDA_VISIBLE_DEVICES") or None if hip_val is not None and cuda_val is not None: if hip_val != cuda_val: raise ValueError( f"Inconsistent GPU visibility env vars: " f"HIP_VISIBLE_DEVICES='{hip_val}' vs " f"CUDA_VISIBLE_DEVICES='{cuda_val}'. " f"Please set only one, or ensure they match." ) elif hip_val is not None: os.environ["CUDA_VISIBLE_DEVICES"] = hip_val # Sync at import time - catches misconfigurations from process start. _sync_hip_cuda_env_vars() # AMDSMI utils # Note that NVML is not affected by `{CUDA/HIP}_VISIBLE_DEVICES`, # all the related functions work on real physical device ids. # the major benefit of using AMDSMI is that it will not initialize CUDA def with_amdsmi_context(fn): @wraps(fn) def wrapper(*args, **kwargs): amdsmi_init() try: return fn(*args, **kwargs) finally: amdsmi_shut_down() return wrapper @with_amdsmi_context def _query_gcn_arch_from_amdsmi() -> str: """Query GCN arch from amdsmi. Raises if not available.""" handles = amdsmi_get_processor_handles() if handles: asic_info = amdsmi_get_gpu_asic_info(handles[0]) # Use target_graphics_version which contains the gfx name # e.g., 'gfx942' for MI300X/MI325X target_gfx = asic_info.get("target_graphics_version", "") if target_gfx: return target_gfx raise RuntimeError("amdsmi did not return valid GCN arch") def _get_gcn_arch() -> str: """Get GCN arch via amdsmi (no CUDA init), fallback to torch.cuda. Called once at module level; result stored in _GCN_ARCH. """ try: return _query_gcn_arch_from_amdsmi() except Exception as e: logger.debug("Failed to get GCN arch via amdsmi: %s", e) # Ultimate fallback: use torch.cuda (will initialize CUDA) return torch.cuda.get_device_properties("cuda").gcnArchName # Resolve once at module load. Uses amdsmi (no CUDA init) so Ray workers # can still set CUDA_VISIBLE_DEVICES after import. # These are plain Python bools — fully torch.compile/Dynamo safe. _GCN_ARCH = _get_gcn_arch() _ON_GFX1X = any(arch in _GCN_ARCH for arch in ["gfx11", "gfx12"]) _ON_GFX11 = "gfx11" in _GCN_ARCH _ON_GFX1100 = "gfx1100" in _GCN_ARCH _ON_GFX1151 = "gfx1151" in _GCN_ARCH _ON_GFX115X = "gfx115" in _GCN_ARCH _ON_GFX12X = any(arch in _GCN_ARCH for arch in ["gfx12"]) _ON_MI3XX = any(arch in _GCN_ARCH for arch in ["gfx942", "gfx950"]) _ON_GFX9 = any(arch in _GCN_ARCH for arch in ["gfx90a", "gfx942", "gfx950"]) _ON_GFX90A = "gfx90a" in _GCN_ARCH _ON_GFX942 = "gfx942" in _GCN_ARCH _ON_GFX950 = "gfx950" in _GCN_ARCH _ON_GFX1250 = "gfx1250" in _GCN_ARCH _ON_CDNA = any(arch in _GCN_ARCH for arch in ["gfx9", "gfx1250"]) # RDNA = gfx11/gfx12 minus the CDNA-classified gfx1250. _ON_RDNA = _ON_GFX1X and not _ON_CDNA _ON_RDNA4 = any(arch in _GCN_ARCH for arch in ["gfx1200", "gfx1201"]) def on_gfx1x() -> bool: return _ON_GFX1X and not _ON_CDNA def on_gfx11() -> bool: return _ON_GFX11 def on_gfx1100() -> bool: return _ON_GFX1100 def on_gfx12x() -> bool: return _ON_GFX12X and not _ON_CDNA def on_gfx1250() -> bool: return _ON_GFX1250 def on_rdna4() -> bool: return _ON_RDNA4 def on_mi3xx() -> bool: return _ON_MI3XX def on_gfx90a() -> bool: return _ON_GFX90A def on_gfx942() -> bool: return _ON_GFX942 def on_gfx950() -> bool: return _ON_GFX950 def get_cdna_version() -> int: if on_gfx90a(): return 2 if on_gfx942(): return 3 if on_gfx950(): return 4 if on_gfx1250(): return 5 return 0 # Enable HIP online tuning early, before hipBLASLt initializes. # Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM. if ( envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_LINEAR and envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM and get_cdna_version() > 2 ): os.environ["HIP_ONLINE_TUNING"] = "1" # RCCL 2.30's RAS thread can abort the process at exit ("terminate called # without an active exception") when its shutdown notification fails. # https://github.com/ROCm/rocm-systems/issues/13572 if torch.version.hip: os.environ.setdefault("NCCL_RAS_ENABLE", "0")