Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-06-20 13:38:10 -07:00
committed by GitHub
co-authored by Claude Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> kourosh hakhamaneshi
parent e9de72fe6c
commit ebfbcfe46a
24 changed files with 722 additions and 185 deletions
+58
View File
@@ -6,6 +6,7 @@ import copy
import dataclasses
import functools
import json
import os
import sys
from collections.abc import Callable
from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass
@@ -465,6 +466,7 @@ class EngineArgs:
numa_bind: bool = ParallelConfig.numa_bind
numa_bind_nodes: list[int] | None = ParallelConfig.numa_bind_nodes
numa_bind_cpus: list[str] | None = ParallelConfig.numa_bind_cpus
device_ids: list[int | str] | None = None
tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size
decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
@@ -979,6 +981,20 @@ class EngineArgs:
parallel_group.add_argument(
"--numa-bind-cpus", **parallel_kwargs["numa_bind_cpus"]
)
parallel_group.add_argument(
"--device-ids",
type=lambda s: [
int(device_id) if device_id.isdigit() else device_id
for device_id in (part.strip() for part in s.split(","))
],
default=None,
help="Comma-separated physical GPU device IDs or UUIDs to use "
'(e.g. --device-ids "2,3,5,7"). Avoids setting '
"CUDA_VISIBLE_DEVICES, preserving full GPU topology "
"visibility for GPU-NIC affinity and DeepGEMM. "
"Note: has no effect with Ray executors; use Ray "
"placement groups for GPU selection instead.",
)
parallel_group.add_argument(
"--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
)
@@ -1716,6 +1732,47 @@ class EngineArgs:
)
return SpeculativeConfig(**self.speculative_config)
def _resolve_device_ids(self) -> list[int] | None:
if not self.device_ids:
return None
if self.distributed_executor_backend == "ray":
logger.warning(
"--device-ids has no effect when using the Ray executor. "
"Use Ray placement groups for GPU selection instead."
)
ids = self.device_ids
if len(set(ids)) != len(ids):
raise ValueError(f"--device-ids must not contain duplicates: {ids}")
if all(isinstance(i, str) for i in ids):
return [
current_platform.device_control_id_to_physical_device_id(i)
for i in cast(list[str], ids)
]
if any(isinstance(i, str) for i in ids):
raise ValueError("--device-ids must not mix integer IDs and UUIDs")
int_ids = cast(list[int], ids)
# Compose with CUDA_VISIBLE_DEVICES: if CVD is set, treat
# --device-ids values as indices into the CVD-visible set.
cvd = getattr(
envs,
current_platform.device_control_env_var,
os.environ.get(current_platform.device_control_env_var),
)
if cvd:
cvd_ids = [
current_platform.device_control_id_to_physical_device_id(x)
for x in cvd.split(",")
]
for i in int_ids:
if i >= len(cvd_ids):
raise ValueError(
f"--device-ids index {i} is out of range for "
f"{current_platform.device_control_env_var}"
f"={cvd} ({len(cvd_ids)} devices visible)"
)
return [cvd_ids[i] for i in int_ids]
return int_ids
def create_diffusion_config(self) -> DiffusionConfig | None:
if self.diffusion_config is None:
return None
@@ -2029,6 +2086,7 @@ class EngineArgs:
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
_api_process_count=self._api_process_count,
_api_process_rank=self._api_process_rank,
assigned_physical_gpu_ids=self._resolve_device_ids(),
numa_bind=self.numa_bind,
numa_bind_nodes=self.numa_bind_nodes,
numa_bind_cpus=self.numa_bind_cpus,