Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-06-20 13:38:10 -07:00
committed by GitHub
co-authored by Claude Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> kourosh hakhamaneshi
parent e9de72fe6c
commit ebfbcfe46a
24 changed files with 722 additions and 185 deletions
+22 -31
View File
@@ -258,30 +258,35 @@ class RayDistributedExecutor(Executor):
}
self.collective_rpc("adjust_rank", args=(rerank_mapping,))
# Get the set of GPU IDs used on each node.
worker_node_and_gpu_ids = []
# Get the set of physical GPU IDs used on each node.
worker_node_and_physical_gpu_ids = []
for worker in [self.driver_dummy_worker] + self.workers:
if worker is None:
# driver_dummy_worker can be None when using ray spmd worker.
continue
worker_node_and_gpu_ids.append(
ray.get(worker.get_node_and_gpu_ids.remote()) # type: ignore[attr-defined]
worker_node_and_physical_gpu_ids.append(
ray.get(worker.get_node_and_physical_gpu_ids.remote()) # type: ignore[attr-defined]
)
node_workers = defaultdict(list) # node id -> list of worker ranks
node_gpus = defaultdict(list) # node id -> list of gpu ids
node_physical_gpu_ids = defaultdict(list) # node id -> physical GPU IDs
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
for i, (node_id, physical_gpu_ids) in enumerate(
worker_node_and_physical_gpu_ids
):
node_workers[node_id].append(i)
# `gpu_ids` can be a list of strings or integers.
# `physical_gpu_ids` can be a list of strings or integers.
# convert them to integers for consistency.
# NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
# NOTE: physical GPU IDs can be larger than 9 (e.g. 16 GPUs),
# string sorting is not sufficient.
# see https://github.com/vllm-project/vllm/issues/5590
gpu_ids = [int(x) for x in gpu_ids]
node_gpus[node_id].extend(gpu_ids)
for node_id, gpu_ids in node_gpus.items():
node_gpus[node_id] = sorted(gpu_ids)
physical_gpu_ids = [
current_platform.device_control_id_to_physical_device_id(str(x))
for x in physical_gpu_ids
]
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
all_ips = set(worker_ips + [driver_ip])
n_ips = len(all_ips)
@@ -297,23 +302,8 @@ class RayDistributedExecutor(Executor):
" each node."
)
# Set environment variables for the driver and workers.
# We set CUDA_VISIBLE_DEVICES to ALL GPUs on the node for each worker.
# This is needed because:
# 1. Ray's compiled DAG needs to find the allocated GPU in
# CUDA_VISIBLE_DEVICES.
# 2. vLLM's communication layer (NCCL, CustomAllreduce) needs to see
# all GPUs for P2P checks and communication setup. Though if it was
# just this reason, we could have also just kept the visible devices
# unset.
# Each worker will use local_rank to index into the visible devices.
all_args_to_update_environment_variables = [
{
current_platform.device_control_env_var: ",".join(
map(str, node_gpus[node_id])
),
}
for (node_id, _) in worker_node_and_gpu_ids
all_args_to_update_environment_variables: list[dict[str, str]] = [
{} for _ in worker_node_and_physical_gpu_ids
]
# Environment variables to copy from driver to workers
@@ -336,7 +326,7 @@ class RayDistributedExecutor(Executor):
"update_environment_variables", args=(self._get_env_vars_to_be_updated(),)
)
if len(node_gpus) == 1:
if len(node_physical_gpu_ids) == 1:
# in single node case, we don't need to get the IP address.
# the loopback address is sufficient
# NOTE: a node may have several IP addresses, one for each
@@ -352,10 +342,11 @@ class RayDistributedExecutor(Executor):
# Initialize the actual workers inside worker wrapper.
all_kwargs = []
for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
for rank, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
local_rank = node_workers[node_id].index(rank)
kwargs = dict(
vllm_config=self.vllm_config,
assigned_physical_gpu_ids=sorted(node_physical_gpu_ids[node_id]),
local_rank=local_rank,
rank=rank,
distributed_init_method=distributed_init_method,