Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+137
-108
@@ -41,7 +41,6 @@ torch.backends.cuda.enable_cudnn_sdp(False)
|
||||
|
||||
|
||||
def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
|
||||
|
||||
@wraps(fn)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
||||
pynvml.nvmlInit()
|
||||
@@ -86,9 +85,7 @@ class CudaPlatformBase(Platform):
|
||||
_ = torch.zeros(1, device=device)
|
||||
|
||||
@classmethod
|
||||
def get_device_capability(cls,
|
||||
device_id: int = 0
|
||||
) -> Optional[DeviceCapability]:
|
||||
def get_device_capability(cls, device_id: int = 0) -> Optional[DeviceCapability]:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@@ -122,8 +119,7 @@ class CudaPlatformBase(Platform):
|
||||
# TODO(lucas): handle this more gracefully
|
||||
# Note: model_config may be None during testing
|
||||
if model_config is not None and model_config.use_mla:
|
||||
use_sparse = hasattr(vllm_config.model_config.hf_config,
|
||||
"index_topk")
|
||||
use_sparse = hasattr(vllm_config.model_config.hf_config, "index_topk")
|
||||
# If `VLLM_ATTENTION_BACKEND` is not set and we are using MLA,
|
||||
# then we default to FlashMLA backend for non-blackwell GPUs,
|
||||
# else we default to CutlassMLA. For each case, we force the
|
||||
@@ -146,43 +142,47 @@ class CudaPlatformBase(Platform):
|
||||
use_flashmla = True
|
||||
else:
|
||||
# Forced case
|
||||
use_flashmla = (envs.VLLM_ATTENTION_BACKEND == "FLASHMLA")
|
||||
use_cutlass_mla = (
|
||||
envs.VLLM_ATTENTION_BACKEND == "CUTLASS_MLA")
|
||||
use_flashinfer_mla = (
|
||||
envs.VLLM_ATTENTION_BACKEND == "FLASHINFER_MLA")
|
||||
use_flashmla = envs.VLLM_ATTENTION_BACKEND == "FLASHMLA"
|
||||
use_cutlass_mla = envs.VLLM_ATTENTION_BACKEND == "CUTLASS_MLA"
|
||||
use_flashinfer_mla = envs.VLLM_ATTENTION_BACKEND == "FLASHINFER_MLA"
|
||||
|
||||
from vllm.attention.ops.flashmla import is_flashmla_supported
|
||||
if use_flashmla and is_flashmla_supported()[0] \
|
||||
and cache_config.block_size != 64:
|
||||
|
||||
if (
|
||||
use_flashmla
|
||||
and is_flashmla_supported()[0]
|
||||
and cache_config.block_size != 64
|
||||
):
|
||||
cache_config.block_size = 64
|
||||
logger.info(
|
||||
"Forcing kv cache block size to 64 for FlashMLA backend.")
|
||||
logger.info("Forcing kv cache block size to 64 for FlashMLA backend.")
|
||||
|
||||
if use_cutlass_mla and cache_config.block_size != 128:
|
||||
cache_config.block_size = 128
|
||||
logger.info("Forcing kv cache block size to 128 for "
|
||||
"CUTLASS_MLA backend.")
|
||||
logger.info(
|
||||
"Forcing kv cache block size to 128 for CUTLASS_MLA backend."
|
||||
)
|
||||
|
||||
if use_flashinfer_mla and cache_config.block_size not in [32, 64]:
|
||||
cache_config.block_size = 64
|
||||
logger.info(
|
||||
"Forcing kv cache block size to 64 for FlashInferMLA "
|
||||
"backend.")
|
||||
"Forcing kv cache block size to 64 for FlashInferMLA backend."
|
||||
)
|
||||
|
||||
# TODO(Chen): remove this hacky code
|
||||
if use_sparse and cache_config.block_size != 64:
|
||||
cache_config.block_size = 64
|
||||
logger.info(
|
||||
"Forcing kv cache block size to 64 for FlashMLASparse "
|
||||
"backend.")
|
||||
"Forcing kv cache block size to 64 for FlashMLASparse backend."
|
||||
)
|
||||
# lazy import to avoid circular import
|
||||
from vllm.config import CUDAGraphMode
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if (envs.VLLM_ALL2ALL_BACKEND == "deepep_high_throughput"
|
||||
and parallel_config.data_parallel_size > 1
|
||||
and compilation_config.cudagraph_mode != CUDAGraphMode.NONE):
|
||||
if (
|
||||
envs.VLLM_ALL2ALL_BACKEND == "deepep_high_throughput"
|
||||
and parallel_config.data_parallel_size > 1
|
||||
and compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
):
|
||||
# TODO: Piecewise Cuda graph might be enabled
|
||||
# if torch compile cache key issue fixed
|
||||
# See https://github.com/vllm-project/vllm/pull/25093
|
||||
@@ -192,20 +192,20 @@ class CudaPlatformBase(Platform):
|
||||
"CUDA Graphs. "
|
||||
"In order to use CUDA Graphs for decode-optimized workloads, "
|
||||
"set VLLM_ALL2ALL_BACKEND to another option, such as "
|
||||
"deepep_low_latency, pplx, or allgather_reducescatter.")
|
||||
"deepep_low_latency, pplx, or allgather_reducescatter."
|
||||
)
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
@classmethod
|
||||
def get_current_memory_usage(cls,
|
||||
device: Optional[torch.types.Device] = None
|
||||
) -> float:
|
||||
def get_current_memory_usage(
|
||||
cls, device: Optional[torch.types.Device] = None
|
||||
) -> float:
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_peak_memory_stats(device)
|
||||
return torch.cuda.max_memory_allocated(device)
|
||||
|
||||
@classmethod
|
||||
def get_vit_attn_backend(cls, head_size: int,
|
||||
dtype: torch.dtype) -> "_Backend":
|
||||
def get_vit_attn_backend(cls, head_size: int, dtype: torch.dtype) -> "_Backend":
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
|
||||
# For Blackwell GPUs, force TORCH_SDPA for now.
|
||||
@@ -217,10 +217,14 @@ class CudaPlatformBase(Platform):
|
||||
return _Backend.XFORMERS
|
||||
|
||||
if cls.has_device_capability(80):
|
||||
FLASH_ATTN_V1 = "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend" # noqa: E501
|
||||
FLASH_ATTN_V1 = (
|
||||
"vllm.v1.attention.backends.flash_attn.FlashAttentionBackend" # noqa: E501
|
||||
)
|
||||
from vllm.attention.selector import is_attn_backend_supported
|
||||
|
||||
is_default_fa_supported = is_attn_backend_supported(
|
||||
FLASH_ATTN_V1, head_size, dtype, allow_import_error=False)
|
||||
FLASH_ATTN_V1, head_size, dtype, allow_import_error=False
|
||||
)
|
||||
if is_default_fa_supported:
|
||||
return _Backend.FLASH_ATTN
|
||||
else:
|
||||
@@ -231,83 +235,109 @@ class CudaPlatformBase(Platform):
|
||||
return _Backend.XFORMERS
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
|
||||
kv_cache_dtype, block_size, use_v1, use_mla,
|
||||
has_sink, use_sparse) -> str:
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
selected_backend,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
use_v1,
|
||||
use_mla,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
) -> str:
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
|
||||
if use_mla:
|
||||
if not use_v1:
|
||||
raise RuntimeError(
|
||||
"MLA attention backends require the V1 engine. "
|
||||
"Set VLLM_USE_V1=1 to enable them.")
|
||||
"Set VLLM_USE_V1=1 to enable them."
|
||||
)
|
||||
|
||||
from vllm.attention.ops.flashmla import is_flashmla_supported
|
||||
from vllm.attention.utils.fa_utils import flash_attn_supports_mla
|
||||
|
||||
if use_sparse:
|
||||
logger.info_once("Using Sparse MLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"FlashMLASparseBackend")
|
||||
return (
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"FlashMLASparseBackend"
|
||||
)
|
||||
|
||||
use_cutlassmla = selected_backend == _Backend.CUTLASS_MLA or (
|
||||
selected_backend is None and cls.is_device_capability(100)
|
||||
and block_size == 128)
|
||||
selected_backend is None
|
||||
and cls.is_device_capability(100)
|
||||
and block_size == 128
|
||||
)
|
||||
use_flashinfermla = selected_backend == _Backend.FLASHINFER_MLA or (
|
||||
selected_backend is None and cls.is_device_capability(100)
|
||||
and block_size in [32, 64])
|
||||
selected_backend is None
|
||||
and cls.is_device_capability(100)
|
||||
and block_size in [32, 64]
|
||||
)
|
||||
use_flashmla = selected_backend == _Backend.FLASHMLA or (
|
||||
selected_backend is None and is_flashmla_supported()[0])
|
||||
selected_backend is None and is_flashmla_supported()[0]
|
||||
)
|
||||
use_flashattn = selected_backend == _Backend.FLASH_ATTN_MLA or (
|
||||
selected_backend is None and flash_attn_supports_mla())
|
||||
selected_backend is None and flash_attn_supports_mla()
|
||||
)
|
||||
use_triton = selected_backend == _Backend.TRITON_MLA or (
|
||||
selected_backend is None)
|
||||
selected_backend is None
|
||||
)
|
||||
|
||||
if use_cutlassmla:
|
||||
logger.info_once("Using Cutlass MLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla."
|
||||
"cutlass_mla.CutlassMLABackend")
|
||||
return "vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend"
|
||||
if use_flashinfermla:
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
set_kv_cache_layout)
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
|
||||
set_kv_cache_layout("HND")
|
||||
logger.info_once("Using FlashInfer MLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla."
|
||||
"flashinfer_mla.FlashInferMLABackend")
|
||||
return (
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend"
|
||||
)
|
||||
if use_flashmla:
|
||||
if block_size != 64:
|
||||
logger.warning(
|
||||
"FlashMLA backend is not supported for block size %d"
|
||||
" (currently only supports block size 64).",
|
||||
block_size)
|
||||
block_size,
|
||||
)
|
||||
else:
|
||||
logger.info_once("Using FlashMLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla."
|
||||
"flashmla.FlashMLABackend")
|
||||
return "vllm.v1.attention.backends.mla.flashmla.FlashMLABackend"
|
||||
if use_flashattn:
|
||||
logger.info_once(
|
||||
"Using FlashAttention MLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla."
|
||||
"flashattn_mla.FlashAttnMLABackend")
|
||||
logger.info_once("Using FlashAttention MLA backend on V1 engine.")
|
||||
return (
|
||||
"vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
|
||||
)
|
||||
if use_triton:
|
||||
logger.info_once("Using Triton MLA backend on V1 engine.")
|
||||
return ("vllm.v1.attention.backends.mla."
|
||||
"triton_mla.TritonMLABackend")
|
||||
return "vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend"
|
||||
if use_v1:
|
||||
FLASHINFER_V1 = "vllm.v1.attention.backends.flashinfer.FlashInferBackend" # noqa: E501
|
||||
FLEX_ATTENTION_V1 = "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend" # noqa: E501
|
||||
TRITON_ATTN = "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend" # noqa: E501
|
||||
FLASH_ATTN_V1 = "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend" # noqa: E501
|
||||
FLEX_ATTENTION_V1 = (
|
||||
"vllm.v1.attention.backends.flex_attention.FlexAttentionBackend" # noqa: E501
|
||||
)
|
||||
TRITON_ATTN = (
|
||||
"vllm.v1.attention.backends.triton_attn.TritonAttentionBackend" # noqa: E501
|
||||
)
|
||||
FLASH_ATTN_V1 = (
|
||||
"vllm.v1.attention.backends.flash_attn.FlashAttentionBackend" # noqa: E501
|
||||
)
|
||||
TREE_ATTN_V1 = "vllm.v1.attention.backends.tree_attn.TreeAttentionBackend" # noqa: E501
|
||||
XFORMERS_V1 = "vllm.v1.attention.backends.xformers.XFormersAttentionBackend" # noqa: E501
|
||||
|
||||
use_fp8_kv_cache = (kv_cache_dtype is not None
|
||||
and kv_cache_dtype.startswith("fp8"))
|
||||
use_fp8_kv_cache = kv_cache_dtype is not None and kv_cache_dtype.startswith(
|
||||
"fp8"
|
||||
)
|
||||
|
||||
if selected_backend == _Backend.FLASHINFER:
|
||||
logger.info_once("Using FlashInfer backend on V1 engine.")
|
||||
if cls.has_device_capability(100):
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
set_kv_cache_layout)
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
|
||||
set_kv_cache_layout("HND")
|
||||
return FLASHINFER_V1
|
||||
elif selected_backend == _Backend.FLEX_ATTENTION:
|
||||
@@ -332,13 +362,14 @@ class CudaPlatformBase(Platform):
|
||||
# Prefer FlashInfer for Blackwell GPUs if installed
|
||||
if cls.is_device_capability(100):
|
||||
if is_default_backend_supported := is_attn_backend_supported(
|
||||
FLASHINFER_V1, head_size, dtype):
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
set_kv_cache_layout)
|
||||
FLASHINFER_V1, head_size, dtype
|
||||
):
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
|
||||
logger.info_once(
|
||||
"Using FlashInfer backend with HND KV cache layout on "
|
||||
"V1 engine by default for Blackwell (SM 10.0) GPUs.")
|
||||
"V1 engine by default for Blackwell (SM 10.0) GPUs."
|
||||
)
|
||||
set_kv_cache_layout("HND")
|
||||
|
||||
return FLASHINFER_V1
|
||||
@@ -347,19 +378,18 @@ class CudaPlatformBase(Platform):
|
||||
logger.warning_once(
|
||||
"FlashInfer failed to import for V1 engine on "
|
||||
"Blackwell (SM 10.0) GPUs; it is recommended to "
|
||||
"install FlashInfer for better performance.")
|
||||
"install FlashInfer for better performance."
|
||||
)
|
||||
|
||||
# FlashAttention is the default for SM 8.0+ GPUs
|
||||
if cls.has_device_capability(80):
|
||||
if (has_sink or
|
||||
use_fp8_kv_cache) and not cls.is_device_capability(90):
|
||||
if (has_sink or use_fp8_kv_cache) and not cls.is_device_capability(90):
|
||||
logger.info_once("Using Triton backend on V1 engine.")
|
||||
return TRITON_ATTN
|
||||
elif is_default_backend_supported := is_attn_backend_supported(
|
||||
FLASH_ATTN_V1, head_size, dtype,
|
||||
allow_import_error=False):
|
||||
logger.info_once("Using Flash Attention backend on "
|
||||
"V1 engine.")
|
||||
FLASH_ATTN_V1, head_size, dtype, allow_import_error=False
|
||||
):
|
||||
logger.info_once("Using Flash Attention backend on V1 engine.")
|
||||
return FLASH_ATTN_V1
|
||||
|
||||
# FlexAttention is the default for older GPUs
|
||||
@@ -377,14 +407,14 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
logger.info_once(
|
||||
"Using FlexAttention backend for %s on V1 engine.",
|
||||
", ".join(f"{k}={v}"
|
||||
for k, v in use_flex_attention_reason.items()),
|
||||
", ".join(f"{k}={v}" for k, v in use_flex_attention_reason.items()),
|
||||
)
|
||||
return FLEX_ATTENTION_V1
|
||||
|
||||
raise RuntimeError(
|
||||
"V0 attention backends have been removed. Set VLLM_USE_V1=1 "
|
||||
"to select a supported backend.")
|
||||
"to select a supported backend."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_punica_wrapper(cls) -> str:
|
||||
@@ -392,7 +422,9 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
@classmethod
|
||||
def get_device_communicator_cls(cls) -> str:
|
||||
return "vllm.distributed.device_communicators.cuda_communicator.CudaCommunicator" # noqa
|
||||
return (
|
||||
"vllm.distributed.device_communicators.cuda_communicator.CudaCommunicator" # noqa
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_fp8(cls) -> bool:
|
||||
@@ -430,8 +462,9 @@ class CudaPlatformBase(Platform):
|
||||
backend_options = ProcessGroupNCCL.Options()
|
||||
backend_options._timeout = timeout
|
||||
|
||||
backend_class = ProcessGroupNCCL(prefix_store, group_rank, group_size,
|
||||
backend_options)
|
||||
backend_class = ProcessGroupNCCL(
|
||||
prefix_store, group_rank, group_size, backend_options
|
||||
)
|
||||
backend_type = ProcessGroup.BackendType.NCCL
|
||||
device = torch.device("cuda")
|
||||
pg._set_default_backend(backend_type)
|
||||
@@ -445,8 +478,9 @@ class CudaPlatformBase(Platform):
|
||||
return cuda_device_count_stateless()
|
||||
|
||||
@classmethod
|
||||
def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str,
|
||||
model_config: "ModelConfig") -> bool:
|
||||
def is_kv_cache_dtype_supported(
|
||||
cls, kv_cache_dtype: str, model_config: "ModelConfig"
|
||||
) -> bool:
|
||||
fp8_attention = kv_cache_dtype.startswith("fp8")
|
||||
attention_backend = envs.VLLM_ATTENTION_BACKEND
|
||||
|
||||
@@ -461,12 +495,10 @@ class CudaPlatformBase(Platform):
|
||||
attention_backend = "FLASHMLA"
|
||||
|
||||
# Only FlashMLA and CUTLASS_MLA support fp8
|
||||
if attention_backend in [
|
||||
"FLASHMLA", "CUTLASS_MLA", "FLASHINFER_MLA"
|
||||
]:
|
||||
if attention_backend in ["FLASHMLA", "CUTLASS_MLA", "FLASHINFER_MLA"]:
|
||||
supported = True
|
||||
else:
|
||||
supported = (not fp8_attention)
|
||||
supported = not fp8_attention
|
||||
else:
|
||||
# Default to FlashAttention
|
||||
if attention_backend is None:
|
||||
@@ -477,8 +509,8 @@ class CudaPlatformBase(Platform):
|
||||
supported = True
|
||||
elif attention_backend == "FLASH_ATTN":
|
||||
if fp8_attention:
|
||||
from vllm.attention.utils.fa_utils import (
|
||||
flash_attn_supports_fp8)
|
||||
from vllm.attention.utils.fa_utils import flash_attn_supports_fp8
|
||||
|
||||
supported = flash_attn_supports_fp8()
|
||||
else:
|
||||
supported = True
|
||||
@@ -506,7 +538,8 @@ class CudaPlatformBase(Platform):
|
||||
"with compute capability of at least 8.0. "
|
||||
f"Your {gpu_name} GPU {compute_str}. "
|
||||
"You can use float16 instead by explicitly setting the "
|
||||
"`dtype` flag in CLI, for example: --dtype=half.")
|
||||
"`dtype` flag in CLI, for example: --dtype=half."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def insert_blocks_to_device(
|
||||
@@ -546,13 +579,10 @@ class CudaPlatformBase(Platform):
|
||||
# all the related functions work on real physical device ids.
|
||||
# the major benefit of using NVML is that it will not initialize CUDA
|
||||
class NvmlCudaPlatform(CudaPlatformBase):
|
||||
|
||||
@classmethod
|
||||
@cache
|
||||
@with_nvml_context
|
||||
def get_device_capability(cls,
|
||||
device_id: int = 0
|
||||
) -> Optional[DeviceCapability]:
|
||||
def get_device_capability(cls, device_id: int = 0) -> Optional[DeviceCapability]:
|
||||
try:
|
||||
physical_device_id = cls.device_id_to_physical_device_id(device_id)
|
||||
handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id)
|
||||
@@ -599,9 +629,7 @@ class NvmlCudaPlatform(CudaPlatformBase):
|
||||
"""
|
||||
query if the set of gpus are fully connected by nvlink (1 hop)
|
||||
"""
|
||||
handles = [
|
||||
pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids
|
||||
]
|
||||
handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids]
|
||||
for i, handle in enumerate(handles):
|
||||
for j, peer_handle in enumerate(handles):
|
||||
if i < j:
|
||||
@@ -616,7 +644,8 @@ class NvmlCudaPlatform(CudaPlatformBase):
|
||||
except pynvml.NVMLError:
|
||||
logger.exception(
|
||||
"NVLink detection failed. This is normal if"
|
||||
" your machine has no NVLink equipped.")
|
||||
" your machine has no NVLink equipped."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -630,11 +659,11 @@ class NvmlCudaPlatform(CudaPlatformBase):
|
||||
def log_warnings(cls):
|
||||
device_ids: int = pynvml.nvmlDeviceGetCount()
|
||||
if device_ids > 1:
|
||||
device_names = [
|
||||
cls._get_physical_device_name(i) for i in range(device_ids)
|
||||
]
|
||||
if (len(set(device_names)) > 1
|
||||
and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"):
|
||||
device_names = [cls._get_physical_device_name(i) for i in range(device_ids)]
|
||||
if (
|
||||
len(set(device_names)) > 1
|
||||
and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"
|
||||
):
|
||||
logger.warning(
|
||||
"Detected different devices in the system: %s. Please"
|
||||
" make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
|
||||
@@ -644,7 +673,6 @@ class NvmlCudaPlatform(CudaPlatformBase):
|
||||
|
||||
|
||||
class NonNvmlCudaPlatform(CudaPlatformBase):
|
||||
|
||||
@classmethod
|
||||
@cache
|
||||
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
|
||||
@@ -664,7 +692,8 @@ class NonNvmlCudaPlatform(CudaPlatformBase):
|
||||
def is_fully_connected(cls, physical_device_ids: list[int]) -> bool:
|
||||
logger.exception(
|
||||
"NVLink detection not possible, as context support was"
|
||||
" not found. Assuming no NVLink available.")
|
||||
" not found. Assuming no NVLink available."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user