Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -79,24 +79,25 @@ class RayWorkerProc(WorkerProc):
|
||||
1. __init__: lightweight setup, stores init args (no device/model init)
|
||||
2. initialize_worker: called after GPU IDs are discovered, completes
|
||||
the full WorkerProc initialization with the correct local_rank and
|
||||
CUDA_VISIBLE_DEVICES.
|
||||
logical-to-physical GPU mapping.
|
||||
|
||||
CUDA_VISIBLE_DEVICES setup flow:
|
||||
GPU assignment flow:
|
||||
|
||||
1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
|
||||
not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
|
||||
2. Each actor is scheduled with a placement group and bundle index; Ray resolves
|
||||
the physical GPU ID for that bundle at placement time.
|
||||
3. After placement, the worker discovers that GPU ID and sets
|
||||
CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
|
||||
3. After placement, the executor discovers each worker's GPU ID and passes the
|
||||
node's logical-to-physical mapping (assigned_physical_gpu_ids) to
|
||||
initialize_worker(); CUDA_VISIBLE_DEVICES is never modified.
|
||||
|
||||
There is no workaround for this unset-and-reset sequence when the placement group
|
||||
is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
|
||||
match the GPU tied to the worker's bundle.
|
||||
Scheduling must complete before the mapping is known when the placement
|
||||
group is externally managed: only then is the GPU tied to the worker's
|
||||
bundle resolved.
|
||||
|
||||
This sequence allows multiple vLLM instances to coexist on the same node:
|
||||
each instance is unaware which physical devices others hold, and the
|
||||
externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
|
||||
externally managed placement group avoids device assignment conflicts
|
||||
by binding workers to specific placement group bundles.
|
||||
"""
|
||||
|
||||
@@ -120,28 +121,33 @@ class RayWorkerProc(WorkerProc):
|
||||
is_driver_worker=is_driver_worker,
|
||||
)
|
||||
|
||||
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, gpu_ids) assigned to this actor by Ray."""
|
||||
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
device_key = current_platform.ray_device_key
|
||||
if not device_key:
|
||||
raise RuntimeError(
|
||||
f"current platform {current_platform.device_name} does not support ray."
|
||||
)
|
||||
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, [int(x) for x in gpu_ids]
|
||||
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, [
|
||||
current_platform.device_control_id_to_physical_device_id(str(x))
|
||||
for x in physical_gpu_ids
|
||||
]
|
||||
|
||||
def initialize_worker(
|
||||
self,
|
||||
local_rank: int,
|
||||
env_vars: dict[str, str],
|
||||
driver_env_vars: dict[str, str] | None = None,
|
||||
assigned_physical_gpu_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Complete initialization after GPU assignment is known.
|
||||
|
||||
*driver_env_vars* are applied with ``setdefault`` — they fill
|
||||
in missing vars but never overwrite node-local values.
|
||||
*env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
|
||||
*env_vars* always overwrite.
|
||||
*assigned_physical_gpu_ids* maps local_rank to physical CUDA device ID.
|
||||
"""
|
||||
if driver_env_vars:
|
||||
for key, value in driver_env_vars.items():
|
||||
@@ -149,6 +155,13 @@ class RayWorkerProc(WorkerProc):
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
vllm_config = self._init_kwargs["vllm_config"]
|
||||
assert isinstance(vllm_config, VllmConfig)
|
||||
vllm_config.parallel_config.assigned_physical_gpu_ids = (
|
||||
assigned_physical_gpu_ids
|
||||
)
|
||||
|
||||
self.local_rank = local_rank
|
||||
super().__init__(
|
||||
local_rank=local_rank,
|
||||
@@ -365,36 +378,48 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
)
|
||||
self.ray_worker_handles.append(handle)
|
||||
|
||||
# Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
|
||||
worker_node_and_gpu_ids = ray.get(
|
||||
[h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
|
||||
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
|
||||
# runtime context.
|
||||
worker_node_and_physical_gpu_ids = ray.get(
|
||||
[
|
||||
h.actor.get_node_and_physical_gpu_ids.remote()
|
||||
for h in self.ray_worker_handles
|
||||
]
|
||||
)
|
||||
|
||||
node_workers: dict[str, list[int]] = defaultdict(list)
|
||||
node_gpus: dict[str, list[int]] = defaultdict(list)
|
||||
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
|
||||
node_physical_gpu_ids: dict[str, list[int]] = defaultdict(list)
|
||||
for i, (node_id, physical_gpu_ids) in enumerate(
|
||||
worker_node_and_physical_gpu_ids
|
||||
):
|
||||
node_workers[node_id].append(i)
|
||||
node_gpus[node_id].extend(gpu_ids)
|
||||
for node_id, gpu_ids in node_gpus.items():
|
||||
node_gpus[node_id] = sorted(gpu_ids)
|
||||
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
|
||||
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
|
||||
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
|
||||
|
||||
# Step 7: Initialize workers with correct local_rank and
|
||||
# CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
|
||||
# this executor on its node; local_rank indexes into that set.
|
||||
# Step 7: Initialize workers with local logical ranks and the
|
||||
# logical-to-physical GPU mapping discovered from Ray placement.
|
||||
init_worker_refs = []
|
||||
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
|
||||
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
|
||||
local_rank = node_workers[node_id].index(i)
|
||||
worker_env_vars = {
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
map(str, node_gpus[node_id])
|
||||
),
|
||||
}
|
||||
assigned_physical_gpu_ids = sorted(node_physical_gpu_ids[node_id])
|
||||
worker_env_vars: dict[str, str] = {}
|
||||
self.ray_worker_handles[i].local_rank = local_rank
|
||||
init_worker_refs.append(
|
||||
self.ray_worker_handles[i].actor.initialize_worker.remote(
|
||||
local_rank, worker_env_vars, self.driver_env_vars
|
||||
local_rank,
|
||||
worker_env_vars,
|
||||
self.driver_env_vars,
|
||||
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
|
||||
)
|
||||
)
|
||||
# Also set on the executor-side config for consistency. The mapping
|
||||
# is per-node, so only do this when all workers share one node.
|
||||
if len(node_physical_gpu_ids) == 1:
|
||||
node_id_0 = worker_node_and_physical_gpu_ids[0][0]
|
||||
self.vllm_config.parallel_config.assigned_physical_gpu_ids = sorted(
|
||||
node_physical_gpu_ids[node_id_0]
|
||||
)
|
||||
ray.get(init_worker_refs)
|
||||
|
||||
# Step 8: Collect response MQ handles
|
||||
|
||||
Reference in New Issue
Block a user