diff --git a/tests/v1/executor/test_vllm_net_devices.py b/tests/v1/executor/test_vllm_net_devices.py new file mode 100644 index 0000000000..366dd001f2 --- /dev/null +++ b/tests/v1/executor/test_vllm_net_devices.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.v1.executor.vllm_net_devices import normalize_pci + + +@pytest.mark.parametrize( + "addr, expected", + [ + ("0000:3f:00.0", (0, 0x3F, 0, 0)), + ("0001:00:00.0", (1, 0, 0, 0)), + ("00000001:00:00.0", (1, 0, 0, 0)), + ("0009:00:00.0", (9, 0, 0, 0)), + ("0105:00:00.0", (0x105, 0, 0, 0)), + ("0000:0a:1f.7", (0, 0x0A, 0x1F, 7)), + ], +) +def test_normalize_pci_full_domain(addr, expected): + assert normalize_pci(addr) == expected + + +@pytest.mark.parametrize( + "addr, expected", + [ + ("01:00.0", (0, 1, 0, 0)), + ("3f:00.0", (0, 0x3F, 0, 0)), + ("40:00.0", (0, 0x40, 0, 0)), + ("ff:1f.7", (0, 0xFF, 0x1F, 7)), + ], +) +def test_normalize_pci_short_form(addr, expected): + assert normalize_pci(addr) == expected + + +def test_normalize_pci_case_insensitive(): + assert normalize_pci("0A:1F.7") == normalize_pci("0a:1f.7") + + +def test_normalize_pci_strips_whitespace(): + assert normalize_pci(" 0001:00:00.0 ") == (1, 0, 0, 0) + + +def test_normalize_pci_strips_0x_prefix(): + assert normalize_pci("0x0001:00:00.0") == (1, 0, 0, 0) + + +def test_normalize_pci_missing_function_raises(): + with pytest.raises(ValueError, match="missing function suffix"): + normalize_pci("0001:00:00") + + +def test_normalize_pci_invalid_function_char_raises(): + with pytest.raises(ValueError, match="invalid PCI function"): + normalize_pci("0001:00:00.z") + + +def test_normalize_pci_too_many_segments_raises(): + with pytest.raises(ValueError, match="invalid PCI BDF"): + normalize_pci("a:b:c:d.0") + + +def test_normalize_pci_bus_out_of_range_raises(): + with pytest.raises(ValueError, match="out of range"): + normalize_pci("0000:1ff:00.0") + + +def test_normalize_pci_device_out_of_range_raises(): + with pytest.raises(ValueError, match="out of range"): + normalize_pci("0000:00:20.0") + + +def test_normalize_pci_empty_string_raises(): + with pytest.raises(ValueError): + normalize_pci("") diff --git a/vllm/envs.py b/vllm/envs.py index 3a3934f3cd..c12e3cae24 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -278,6 +278,8 @@ if TYPE_CHECKING: VLLM_XPU_ENABLE_XPU_GRAPH: bool = False VLLM_XPU_USE_SAMPLER_KERNEL: bool = True VLLM_LORA_ENABLE_DUAL_STREAM: bool = False + VLLM_GPU_NIC_PCIE_MAPPING: str = "" + VLLM_NIC_SELECTION_VARS: str = "" def get_default_cache_root(): @@ -1973,6 +1975,13 @@ environment_variables: dict[str, Callable[[], Any]] = { # If set to 1, use Python spinloop extension to poll in a more efficient # way when using the mp backend. "VLLM_USE_SPINLOOP_EXT": lambda: bool(int(os.getenv("VLLM_USE_SPINLOOP_EXT", "0"))), + # Comma-separated GPU_BDF=NIC_BDF pairs for RDMA NIC selection. + # Must be set together with VLLM_NIC_SELECTION_VARS. + "VLLM_GPU_NIC_PCIE_MAPPING": lambda: os.getenv("VLLM_GPU_NIC_PCIE_MAPPING", ""), + # Comma-separated list of env vars to set from the GPU-NIC mapping. + # Each entry is VAR_NAME or VAR_NAME: (suffix appended to + # RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING. + "VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""), } diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 58cef2ec97..57814d29be 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -809,6 +809,22 @@ class NvmlCudaPlatform(CudaPlatformBase): logger.warning("Failed to get NUMA nodes for GPUs: %s", e) return None + @classmethod + @with_nvml_context + def get_all_gpu_pci_bus_ids(cls) -> dict[int, str]: + """Query NVML for GPU index -> PCI bus ID mapping.""" + out: dict[int, str] = {} + for idx in range(pynvml.nvmlDeviceGetCount()): + handle = pynvml.nvmlDeviceGetHandleByIndex(idx) + pci_info = pynvml.nvmlDeviceGetPciInfo(handle) + bus_id = pci_info.busId + if isinstance(bus_id, bytes): + bus_id = bus_id.decode("utf-8") + out[idx] = bus_id.rstrip("\x00") + if not out: + raise RuntimeError("NVML returned no GPU PCI bus ID rows") + return out + @classmethod @with_nvml_context def log_warnings(cls): diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index cf774b7bda..abec3bbe88 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -381,6 +381,19 @@ class Platform: """Get the total memory of a device in bytes.""" raise NotImplementedError + @classmethod + def get_all_gpu_pci_bus_ids(cls) -> dict[int, str]: + """Return a mapping of device index to PCI bus ID string. + + Used by ``VLLM_GPU_NIC_PCIE_MAPPING`` for RDMA NIC selection. + Subclasses should override with platform-specific discovery + (e.g. pynvml for CUDA). + """ + raise NotImplementedError( + "VLLM_GPU_NIC_PCIE_MAPPING is not supported on the " + f"current platform ({cls.device_name})" + ) + @classmethod def inference_mode(cls): """A device-specific wrapper of `torch.inference_mode`. diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index db21d7cee7..449fbdd973 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -60,6 +60,7 @@ from vllm.utils.system_utils import ( ) from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.executor.abstract import Executor, FailureCallback +from vllm.v1.executor.vllm_net_devices import set_worker_net_device from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput from vllm.v1.worker.worker_base import WorkerWrapperBase @@ -811,6 +812,9 @@ class WorkerProc: signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) + # Set net device env vars for the worker if VLLM_GPU_NIC_PCIE_MAPPING is set + set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"]) + worker = None ready_writer = kwargs.pop("ready_pipe") death_pipe = kwargs.pop("death_pipe", None) diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index 92e668406f..c3be3300fd 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -16,6 +16,7 @@ from vllm.platforms import current_platform from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.executor.abstract import Executor +from vllm.v1.executor.vllm_net_devices import set_worker_net_device from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput from vllm.v1.serial_utils import run_method from vllm.v1.worker.worker_base import WorkerWrapperBase @@ -56,6 +57,9 @@ class UniProcExecutor(Executor): shared_worker_lock=Lock(), ) + # Set net device env vars for the worker if VLLM_GPU_NIC_PCIE_MAPPING is set + set_worker_net_device(local_rank, self.vllm_config) + self.driver_worker.init_worker(all_kwargs=[kwargs]) self.driver_worker.init_device() diff --git a/vllm/v1/executor/vllm_net_devices.py b/vllm/v1/executor/vllm_net_devices.py new file mode 100644 index 0000000000..be442f35ee --- /dev/null +++ b/vllm/v1/executor/vllm_net_devices.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU-to-NIC net-device mapping for RDMA transports (UCX, NVSHMEM, ...). + +Shared by both UniProcExecutor (TP=1) and MultiprocExecutor (TP>1). +All transport-specific env vars and sysfs lookups live here so executor +files only need to call ``set_worker_net_device(local_rank, vllm_config)``. + +Requires two env vars set together: +- ``VLLM_GPU_NIC_PCIE_MAPPING`` -- comma-separated GPU_BDF=NIC_BDF pairs. +- ``VLLM_NIC_SELECTION_VARS`` -- comma-separated list of env vars to set, + each optionally suffixed (e.g. ``UCX_NET_DEVICES:1,NCCL_IB_HCA:1``). +""" + +import os +from pathlib import Path + +import vllm.envs as envs +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.platforms import current_platform + +logger = init_logger(__name__) + + +def normalize_pci(addr: str) -> tuple[int, int, int, int]: + """Parse PCI BDF/domain-bus-device-function into comparable ints (all hex). + + Supported shapes: + - ``domain:bus:dev.fn`` -- domain width varies (e.g. ``00000001:00:00.0``, + ``0001:00:00.0``, ``0000:3f:00.0``). + - ``bus:dev.fn`` -- domain **0** (e.g. ``01:00.0``, ``40:00.0``). + + Function suffix is hex (typically ``0``--``7``). Raises ``ValueError`` if malformed. + """ + s = addr.strip().lower().replace(" ", "") + if s.startswith("0x"): + s = s[2:] + if "." not in s: + raise ValueError(f"invalid PCI BDF (missing function suffix): {addr!r}") + body, fn_s = s.rsplit(".", 1) + if not fn_s or any(c not in "0123456789abcdef" for c in fn_s): + raise ValueError(f"invalid PCI function in BDF: {addr!r}") + fn = int(fn_s, 16) + if fn > 0xFF: + raise ValueError(f"PCI function out of range: {addr!r}") + + parts = body.split(":") + if len(parts) == 2: + domain = 0 + bus = int(parts[0], 16) + device = int(parts[1], 16) + elif len(parts) == 3: + domain = int(parts[0], 16) + bus = int(parts[1], 16) + device = int(parts[2], 16) + else: + raise ValueError( + f"invalid PCI BDF (want domain:bus:dev.fn or bus:dev.fn): {addr!r}" + ) + + if bus > 0xFF or device > 0x1F: + raise ValueError(f"PCI bus or device out of range: {addr!r}") + return (domain, bus, device, fn) + + +def parse_gpu_nic_mapping( + raw: str, +) -> dict[tuple[int, int, int, int], tuple[int, int, int, int]]: + out: dict[tuple[int, int, int, int], tuple[int, int, int, int]] = {} + for segment in raw.split(","): + segment = segment.strip() + if not segment: + continue + if "=" not in segment: + raise ValueError( + "VLLM_GPU_NIC_PCIE_MAPPING: expected comma-separated" + f" gpu_bdf=nic_bdf pairs; ambiguous segment: {segment!r}" + ) + gpu_s, nic_s = segment.split("=", 1) + gpu_key = normalize_pci(gpu_s.strip()) + nic_val = normalize_pci(nic_s.strip()) + out[gpu_key] = nic_val + return out + + +def rdma_name_for_nic_pci(nic_pci: tuple[int, int, int, int]) -> str: + """Map NIC PCI BDF to sysfs RDMA name (mlx5_*, ibp*, ...). + + Under ``/sys/class/infiniband//``, ``device`` is a **symlink** to the PCI + device directory (e.g. ``.../0101:00:00.0``). We take ``Path(...).resolve().name`` + as the BDF string. + + ``VLLM_GPU_NIC_PCIE_MAPPING`` NIC keys must **normalize** (via ``normalize_pci``) + to the same tuple as this basename. + """ + ib = Path("/sys/class/infiniband") + if not ib.is_dir(): + raise RuntimeError("/sys/class/infiniband not found or not a directory") + names = sorted(p.name for p in ib.iterdir() if p.is_dir()) + for name in names: + dev_link = ib / name / "device" + if not dev_link.exists(): + continue + try: + resolved = dev_link.resolve() + except OSError: + continue + pci_name = resolved.name + try: + if normalize_pci(pci_name) == nic_pci: + return name + except ValueError: + continue + raise RuntimeError( + f"No /sys/class/infiniband device for NIC PCI {nic_pci}; have entries: {names}" + ) + + +def parse_nic_selection_vars(raw: str) -> list[tuple[str, str]]: + """Parse ``VLLM_NIC_SELECTION_VARS`` into ``(env_var_name, suffix)`` pairs. + + Each entry is ``VAR_NAME`` or ``VAR_NAME:``. The colon and + everything after it is appended verbatim to the RDMA device name. + """ + result: list[tuple[str, str]] = [] + for entry in raw.split(","): + entry = entry.strip() + if not entry: + continue + if ":" in entry: + var_name, suffix = entry.split(":", 1) + result.append((var_name, ":" + suffix)) + else: + result.append((entry, "")) + return result + + +def set_worker_gpu_nic_mapping(local_rank: int) -> None: + """Set NIC selection env vars from VLLM_GPU_NIC_PCIE_MAPPING for a worker. + + Which env vars are set is controlled by ``VLLM_NIC_SELECTION_VARS``. + """ + raw = envs.VLLM_GPU_NIC_PCIE_MAPPING.strip() + if not raw: + return + selection_raw = envs.VLLM_NIC_SELECTION_VARS.strip() + selection_vars = parse_nic_selection_vars(selection_raw) + mapping = parse_gpu_nic_mapping(raw) + pci_by_index = current_platform.get_all_gpu_pci_bus_ids() + # Translate CUDA-relative local_rank to the physical device index, + # which accounts for CUDA_VISIBLE_DEVICES narrowing (e.g. DP sharding). + physical_id = current_platform.device_id_to_physical_device_id(local_rank) + if physical_id not in pci_by_index: + raise RuntimeError( + f"No GPU PCI for physical device index {physical_id} " + f"(local_rank={local_rank}) in map " + f"(have indices {sorted(pci_by_index.keys())})" + ) + gpu_bdf = pci_by_index[physical_id] + gpu_key = normalize_pci(gpu_bdf) + if gpu_key not in mapping: + keys_fmt = ", ".join( + f"{d:04x}:{b:02x}:{dev:02x}.{fn}" + for d, b, dev, fn in sorted(mapping.keys()) + ) + raise RuntimeError( + f"No VLLM_GPU_NIC_PCIE_MAPPING entry for GPU PCI {gpu_bdf} " + f"(worker local_rank={local_rank}); mapped GPUs: {keys_fmt}" + ) + nic_pci = mapping[gpu_key] + rdma_dev = rdma_name_for_nic_pci(nic_pci) + + set_vars: list[str] = [] + for var_name, suffix in selection_vars: + value = f"{rdma_dev}{suffix}" + existing = os.environ.get(var_name, "").strip() + if existing: + value = f"{value},{existing}" + os.environ[var_name] = value + set_vars.append(f"{var_name}={value}") + + nic_fmt = f"{nic_pci[0]:04x}:{nic_pci[1]:02x}:{nic_pci[2]:02x}.{nic_pci[3]}" + logger.info( + "GPU rank %s (PCIe addr %s) mapped to NIC %s (PCIe addr %s) via env vars: %s", + local_rank, + gpu_bdf, + rdma_dev, + nic_fmt, + ", ".join(set_vars), + ) + + +def _dp_adjusted_local_rank(tp_local_rank: int, vllm_config: VllmConfig) -> int: + """Compute the node-wide GPU index accounting for data parallelism. + + On CUDA-alike platforms without env-var device isolation (the common + MP-backend path), the worker sees *all* GPUs on the node and selects + its device via ``torch.accelerator.set_device_index()`` using:: + + dp_local_rank * tp_pp_world_size + tp_local_rank + + This mirrors the adjustment in ``Worker.init_device()`` so we resolve + the correct GPU PCI address *before* the CUDA device is initialised. + """ + pc = vllm_config.parallel_config + if ( + pc.distributed_executor_backend not in ("ray", "external_launcher") + and pc.data_parallel_backend != "ray" + and pc.nnodes_within_dp == 1 + ): + dp_local_rank = pc.data_parallel_rank_local + if dp_local_rank is None: + dp_local_rank = pc.data_parallel_index + tp_pp_world_size = pc.pipeline_parallel_size * pc.tensor_parallel_size + return dp_local_rank * tp_pp_world_size + tp_local_rank + return tp_local_rank + + +def set_worker_net_device(local_rank: int, vllm_config: VllmConfig) -> None: + """Top-level entry point for both UniProcExecutor and MultiprocExecutor. + + Sets NIC selection env vars from ``VLLM_GPU_NIC_PCIE_MAPPING`` and + ``VLLM_NIC_SELECTION_VARS`` if present; no-op otherwise. + """ + has_pcie_mapping = bool(envs.VLLM_GPU_NIC_PCIE_MAPPING.strip()) + has_selection_vars = bool(envs.VLLM_NIC_SELECTION_VARS.strip()) + if has_pcie_mapping and not has_selection_vars: + raise RuntimeError( + "VLLM_GPU_NIC_PCIE_MAPPING is set but VLLM_NIC_SELECTION_VARS " + "is not; both must be set together." + ) + if has_selection_vars and not has_pcie_mapping: + raise RuntimeError( + "VLLM_NIC_SELECTION_VARS is set but VLLM_GPU_NIC_PCIE_MAPPING " + "is not; both must be set together." + ) + # No-op when neither env var is present. + if not has_pcie_mapping and not has_selection_vars: + return + # Both env vars are present, so set the NIC selection env vars. + adjusted_rank = _dp_adjusted_local_rank(local_rank, vllm_config) + set_worker_gpu_nic_mapping(adjusted_rank)