Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-06-20 13:38:10 -07:00
committed by GitHub
co-authored by Claude Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> kourosh hakhamaneshi
parent e9de72fe6c
commit ebfbcfe46a
24 changed files with 722 additions and 185 deletions
+10
View File
@@ -826,6 +826,16 @@ class WorkerProc:
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
# Publish the logical-to-physical mapping early so topology helpers
# work before init_device (needed by set_worker_net_device below).
assigned_physical_gpu_ids = kwargs[
"vllm_config"
].parallel_config.assigned_physical_gpu_ids
if assigned_physical_gpu_ids is not None:
from vllm.platforms.interface import set_assigned_physical_gpu_ids
set_assigned_physical_gpu_ids(assigned_physical_gpu_ids)
# Set net device env vars for the worker if VLLM_GPU_NIC_PCIE_MAPPING is set
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
+22 -31
View File
@@ -258,30 +258,35 @@ class RayDistributedExecutor(Executor):
}
self.collective_rpc("adjust_rank", args=(rerank_mapping,))
# Get the set of GPU IDs used on each node.
worker_node_and_gpu_ids = []
# Get the set of physical GPU IDs used on each node.
worker_node_and_physical_gpu_ids = []
for worker in [self.driver_dummy_worker] + self.workers:
if worker is None:
# driver_dummy_worker can be None when using ray spmd worker.
continue
worker_node_and_gpu_ids.append(
ray.get(worker.get_node_and_gpu_ids.remote()) # type: ignore[attr-defined]
worker_node_and_physical_gpu_ids.append(
ray.get(worker.get_node_and_physical_gpu_ids.remote()) # type: ignore[attr-defined]
)
node_workers = defaultdict(list) # node id -> list of worker ranks
node_gpus = defaultdict(list) # node id -> list of gpu ids
node_physical_gpu_ids = defaultdict(list) # node id -> physical GPU IDs
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
for i, (node_id, physical_gpu_ids) in enumerate(
worker_node_and_physical_gpu_ids
):
node_workers[node_id].append(i)
# `gpu_ids` can be a list of strings or integers.
# `physical_gpu_ids` can be a list of strings or integers.
# convert them to integers for consistency.
# NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
# NOTE: physical GPU IDs can be larger than 9 (e.g. 16 GPUs),
# string sorting is not sufficient.
# see https://github.com/vllm-project/vllm/issues/5590
gpu_ids = [int(x) for x in gpu_ids]
node_gpus[node_id].extend(gpu_ids)
for node_id, gpu_ids in node_gpus.items():
node_gpus[node_id] = sorted(gpu_ids)
physical_gpu_ids = [
current_platform.device_control_id_to_physical_device_id(str(x))
for x in physical_gpu_ids
]
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
all_ips = set(worker_ips + [driver_ip])
n_ips = len(all_ips)
@@ -297,23 +302,8 @@ class RayDistributedExecutor(Executor):
" each node."
)
# Set environment variables for the driver and workers.
# We set CUDA_VISIBLE_DEVICES to ALL GPUs on the node for each worker.
# This is needed because:
# 1. Ray's compiled DAG needs to find the allocated GPU in
# CUDA_VISIBLE_DEVICES.
# 2. vLLM's communication layer (NCCL, CustomAllreduce) needs to see
# all GPUs for P2P checks and communication setup. Though if it was
# just this reason, we could have also just kept the visible devices
# unset.
# Each worker will use local_rank to index into the visible devices.
all_args_to_update_environment_variables = [
{
current_platform.device_control_env_var: ",".join(
map(str, node_gpus[node_id])
),
}
for (node_id, _) in worker_node_and_gpu_ids
all_args_to_update_environment_variables: list[dict[str, str]] = [
{} for _ in worker_node_and_physical_gpu_ids
]
# Environment variables to copy from driver to workers
@@ -336,7 +326,7 @@ class RayDistributedExecutor(Executor):
"update_environment_variables", args=(self._get_env_vars_to_be_updated(),)
)
if len(node_gpus) == 1:
if len(node_physical_gpu_ids) == 1:
# in single node case, we don't need to get the IP address.
# the loopback address is sufficient
# NOTE: a node may have several IP addresses, one for each
@@ -352,10 +342,11 @@ class RayDistributedExecutor(Executor):
# Initialize the actual workers inside worker wrapper.
all_kwargs = []
for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
for rank, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
local_rank = node_workers[node_id].index(rank)
kwargs = dict(
vllm_config=self.vllm_config,
assigned_physical_gpu_ids=sorted(node_physical_gpu_ids[node_id]),
local_rank=local_rank,
rank=rank,
distributed_init_method=distributed_init_method,
+56 -31
View File
@@ -79,24 +79,25 @@ class RayWorkerProc(WorkerProc):
1. __init__: lightweight setup, stores init args (no device/model init)
2. initialize_worker: called after GPU IDs are discovered, completes
the full WorkerProc initialization with the correct local_rank and
CUDA_VISIBLE_DEVICES.
logical-to-physical GPU mapping.
CUDA_VISIBLE_DEVICES setup flow:
GPU assignment flow:
1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
2. Each actor is scheduled with a placement group and bundle index; Ray resolves
the physical GPU ID for that bundle at placement time.
3. After placement, the worker discovers that GPU ID and sets
CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
3. After placement, the executor discovers each worker's GPU ID and passes the
node's logical-to-physical mapping (assigned_physical_gpu_ids) to
initialize_worker(); CUDA_VISIBLE_DEVICES is never modified.
There is no workaround for this unset-and-reset sequence when the placement group
is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
match the GPU tied to the worker's bundle.
Scheduling must complete before the mapping is known when the placement
group is externally managed: only then is the GPU tied to the worker's
bundle resolved.
This sequence allows multiple vLLM instances to coexist on the same node:
each instance is unaware which physical devices others hold, and the
externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
externally managed placement group avoids device assignment conflicts
by binding workers to specific placement group bundles.
"""
@@ -120,28 +121,33 @@ class RayWorkerProc(WorkerProc):
is_driver_worker=is_driver_worker,
)
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
"""Return (node_id, gpu_ids) assigned to this actor by Ray."""
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
node_id = ray.get_runtime_context().get_node_id()
device_key = current_platform.ray_device_key
if not device_key:
raise RuntimeError(
f"current platform {current_platform.device_name} does not support ray."
)
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
return node_id, [int(x) for x in gpu_ids]
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
return node_id, [
current_platform.device_control_id_to_physical_device_id(str(x))
for x in physical_gpu_ids
]
def initialize_worker(
self,
local_rank: int,
env_vars: dict[str, str],
driver_env_vars: dict[str, str] | None = None,
assigned_physical_gpu_ids: list[int] | None = None,
) -> None:
"""Complete initialization after GPU assignment is known.
*driver_env_vars* are applied with ``setdefault`` — they fill
in missing vars but never overwrite node-local values.
*env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
*env_vars* always overwrite.
*assigned_physical_gpu_ids* maps local_rank to physical CUDA device ID.
"""
if driver_env_vars:
for key, value in driver_env_vars.items():
@@ -149,6 +155,13 @@ class RayWorkerProc(WorkerProc):
for key, value in env_vars.items():
os.environ[key] = value
if assigned_physical_gpu_ids is not None:
vllm_config = self._init_kwargs["vllm_config"]
assert isinstance(vllm_config, VllmConfig)
vllm_config.parallel_config.assigned_physical_gpu_ids = (
assigned_physical_gpu_ids
)
self.local_rank = local_rank
super().__init__(
local_rank=local_rank,
@@ -365,36 +378,48 @@ class RayExecutorV2(MultiprocExecutor):
)
self.ray_worker_handles.append(handle)
# Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
worker_node_and_gpu_ids = ray.get(
[h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
# runtime context.
worker_node_and_physical_gpu_ids = ray.get(
[
h.actor.get_node_and_physical_gpu_ids.remote()
for h in self.ray_worker_handles
]
)
node_workers: dict[str, list[int]] = defaultdict(list)
node_gpus: dict[str, list[int]] = defaultdict(list)
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
node_physical_gpu_ids: dict[str, list[int]] = defaultdict(list)
for i, (node_id, physical_gpu_ids) in enumerate(
worker_node_and_physical_gpu_ids
):
node_workers[node_id].append(i)
node_gpus[node_id].extend(gpu_ids)
for node_id, gpu_ids in node_gpus.items():
node_gpus[node_id] = sorted(gpu_ids)
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
# Step 7: Initialize workers with correct local_rank and
# CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
# this executor on its node; local_rank indexes into that set.
# Step 7: Initialize workers with local logical ranks and the
# logical-to-physical GPU mapping discovered from Ray placement.
init_worker_refs = []
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
local_rank = node_workers[node_id].index(i)
worker_env_vars = {
current_platform.device_control_env_var: ",".join(
map(str, node_gpus[node_id])
),
}
assigned_physical_gpu_ids = sorted(node_physical_gpu_ids[node_id])
worker_env_vars: dict[str, str] = {}
self.ray_worker_handles[i].local_rank = local_rank
init_worker_refs.append(
self.ray_worker_handles[i].actor.initialize_worker.remote(
local_rank, worker_env_vars, self.driver_env_vars
local_rank,
worker_env_vars,
self.driver_env_vars,
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
)
)
# Also set on the executor-side config for consistency. The mapping
# is per-node, so only do this when all workers share one node.
if len(node_physical_gpu_ids) == 1:
node_id_0 = worker_node_and_physical_gpu_ids[0][0]
self.vllm_config.parallel_config.assigned_physical_gpu_ids = sorted(
node_physical_gpu_ids[node_id_0]
)
ray.get(init_worker_refs)
# Step 8: Collect response MQ handles
+5 -3
View File
@@ -93,7 +93,7 @@ try:
def get_node_ip(self) -> str:
return get_ip()
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
node_id = ray.get_runtime_context().get_node_id()
device_key = vllm.platforms.current_platform.ray_device_key
if not device_key:
@@ -101,8 +101,10 @@ try:
"current platform %s does not support ray.",
vllm.platforms.current_platform.device_name,
)
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
return node_id, gpu_ids
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[
device_key
]
return node_id, physical_gpu_ids
def setup_device_if_necessary(self):
# TODO(swang): This is needed right now because Ray CG executes