Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-06-20 13:38:10 -07:00
committed by GitHub
co-authored by Claude Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> kourosh hakhamaneshi
parent e9de72fe6c
commit ebfbcfe46a
24 changed files with 722 additions and 185 deletions
+56 -31
View File
@@ -79,24 +79,25 @@ class RayWorkerProc(WorkerProc):
1. __init__: lightweight setup, stores init args (no device/model init)
2. initialize_worker: called after GPU IDs are discovered, completes
the full WorkerProc initialization with the correct local_rank and
CUDA_VISIBLE_DEVICES.
logical-to-physical GPU mapping.
CUDA_VISIBLE_DEVICES setup flow:
GPU assignment flow:
1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
2. Each actor is scheduled with a placement group and bundle index; Ray resolves
the physical GPU ID for that bundle at placement time.
3. After placement, the worker discovers that GPU ID and sets
CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
3. After placement, the executor discovers each worker's GPU ID and passes the
node's logical-to-physical mapping (assigned_physical_gpu_ids) to
initialize_worker(); CUDA_VISIBLE_DEVICES is never modified.
There is no workaround for this unset-and-reset sequence when the placement group
is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
match the GPU tied to the worker's bundle.
Scheduling must complete before the mapping is known when the placement
group is externally managed: only then is the GPU tied to the worker's
bundle resolved.
This sequence allows multiple vLLM instances to coexist on the same node:
each instance is unaware which physical devices others hold, and the
externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
externally managed placement group avoids device assignment conflicts
by binding workers to specific placement group bundles.
"""
@@ -120,28 +121,33 @@ class RayWorkerProc(WorkerProc):
is_driver_worker=is_driver_worker,
)
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
"""Return (node_id, gpu_ids) assigned to this actor by Ray."""
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
node_id = ray.get_runtime_context().get_node_id()
device_key = current_platform.ray_device_key
if not device_key:
raise RuntimeError(
f"current platform {current_platform.device_name} does not support ray."
)
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
return node_id, [int(x) for x in gpu_ids]
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
return node_id, [
current_platform.device_control_id_to_physical_device_id(str(x))
for x in physical_gpu_ids
]
def initialize_worker(
self,
local_rank: int,
env_vars: dict[str, str],
driver_env_vars: dict[str, str] | None = None,
assigned_physical_gpu_ids: list[int] | None = None,
) -> None:
"""Complete initialization after GPU assignment is known.
*driver_env_vars* are applied with ``setdefault`` — they fill
in missing vars but never overwrite node-local values.
*env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
*env_vars* always overwrite.
*assigned_physical_gpu_ids* maps local_rank to physical CUDA device ID.
"""
if driver_env_vars:
for key, value in driver_env_vars.items():
@@ -149,6 +155,13 @@ class RayWorkerProc(WorkerProc):
for key, value in env_vars.items():
os.environ[key] = value
if assigned_physical_gpu_ids is not None:
vllm_config = self._init_kwargs["vllm_config"]
assert isinstance(vllm_config, VllmConfig)
vllm_config.parallel_config.assigned_physical_gpu_ids = (
assigned_physical_gpu_ids
)
self.local_rank = local_rank
super().__init__(
local_rank=local_rank,
@@ -365,36 +378,48 @@ class RayExecutorV2(MultiprocExecutor):
)
self.ray_worker_handles.append(handle)
# Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
worker_node_and_gpu_ids = ray.get(
[h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
# runtime context.
worker_node_and_physical_gpu_ids = ray.get(
[
h.actor.get_node_and_physical_gpu_ids.remote()
for h in self.ray_worker_handles
]
)
node_workers: dict[str, list[int]] = defaultdict(list)
node_gpus: dict[str, list[int]] = defaultdict(list)
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
node_physical_gpu_ids: dict[str, list[int]] = defaultdict(list)
for i, (node_id, physical_gpu_ids) in enumerate(
worker_node_and_physical_gpu_ids
):
node_workers[node_id].append(i)
node_gpus[node_id].extend(gpu_ids)
for node_id, gpu_ids in node_gpus.items():
node_gpus[node_id] = sorted(gpu_ids)
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
# Step 7: Initialize workers with correct local_rank and
# CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
# this executor on its node; local_rank indexes into that set.
# Step 7: Initialize workers with local logical ranks and the
# logical-to-physical GPU mapping discovered from Ray placement.
init_worker_refs = []
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
local_rank = node_workers[node_id].index(i)
worker_env_vars = {
current_platform.device_control_env_var: ",".join(
map(str, node_gpus[node_id])
),
}
assigned_physical_gpu_ids = sorted(node_physical_gpu_ids[node_id])
worker_env_vars: dict[str, str] = {}
self.ray_worker_handles[i].local_rank = local_rank
init_worker_refs.append(
self.ray_worker_handles[i].actor.initialize_worker.remote(
local_rank, worker_env_vars, self.driver_env_vars
local_rank,
worker_env_vars,
self.driver_env_vars,
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
)
)
# Also set on the executor-side config for consistency. The mapping
# is per-node, so only do this when all workers share one node.
if len(node_physical_gpu_ids) == 1:
node_id_0 = worker_node_and_physical_gpu_ids[0][0]
self.vllm_config.parallel_config.assigned_physical_gpu_ids = sorted(
node_physical_gpu_ids[node_id_0]
)
ray.get(init_worker_refs)
# Step 8: Collect response MQ handles