Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -826,6 +826,16 @@ class WorkerProc:
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
# Publish the logical-to-physical mapping early so topology helpers
|
||||
# work before init_device (needed by set_worker_net_device below).
|
||||
assigned_physical_gpu_ids = kwargs[
|
||||
"vllm_config"
|
||||
].parallel_config.assigned_physical_gpu_ids
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
from vllm.platforms.interface import set_assigned_physical_gpu_ids
|
||||
|
||||
set_assigned_physical_gpu_ids(assigned_physical_gpu_ids)
|
||||
|
||||
# Set net device env vars for the worker if VLLM_GPU_NIC_PCIE_MAPPING is set
|
||||
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
|
||||
|
||||
|
||||
@@ -258,30 +258,35 @@ class RayDistributedExecutor(Executor):
|
||||
}
|
||||
self.collective_rpc("adjust_rank", args=(rerank_mapping,))
|
||||
|
||||
# Get the set of GPU IDs used on each node.
|
||||
worker_node_and_gpu_ids = []
|
||||
# Get the set of physical GPU IDs used on each node.
|
||||
worker_node_and_physical_gpu_ids = []
|
||||
for worker in [self.driver_dummy_worker] + self.workers:
|
||||
if worker is None:
|
||||
# driver_dummy_worker can be None when using ray spmd worker.
|
||||
continue
|
||||
worker_node_and_gpu_ids.append(
|
||||
ray.get(worker.get_node_and_gpu_ids.remote()) # type: ignore[attr-defined]
|
||||
worker_node_and_physical_gpu_ids.append(
|
||||
ray.get(worker.get_node_and_physical_gpu_ids.remote()) # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
node_workers = defaultdict(list) # node id -> list of worker ranks
|
||||
node_gpus = defaultdict(list) # node id -> list of gpu ids
|
||||
node_physical_gpu_ids = defaultdict(list) # node id -> physical GPU IDs
|
||||
|
||||
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
|
||||
for i, (node_id, physical_gpu_ids) in enumerate(
|
||||
worker_node_and_physical_gpu_ids
|
||||
):
|
||||
node_workers[node_id].append(i)
|
||||
# `gpu_ids` can be a list of strings or integers.
|
||||
# `physical_gpu_ids` can be a list of strings or integers.
|
||||
# convert them to integers for consistency.
|
||||
# NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
|
||||
# NOTE: physical GPU IDs can be larger than 9 (e.g. 16 GPUs),
|
||||
# string sorting is not sufficient.
|
||||
# see https://github.com/vllm-project/vllm/issues/5590
|
||||
gpu_ids = [int(x) for x in gpu_ids]
|
||||
node_gpus[node_id].extend(gpu_ids)
|
||||
for node_id, gpu_ids in node_gpus.items():
|
||||
node_gpus[node_id] = sorted(gpu_ids)
|
||||
physical_gpu_ids = [
|
||||
current_platform.device_control_id_to_physical_device_id(str(x))
|
||||
for x in physical_gpu_ids
|
||||
]
|
||||
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
|
||||
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
|
||||
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
|
||||
|
||||
all_ips = set(worker_ips + [driver_ip])
|
||||
n_ips = len(all_ips)
|
||||
@@ -297,23 +302,8 @@ class RayDistributedExecutor(Executor):
|
||||
" each node."
|
||||
)
|
||||
|
||||
# Set environment variables for the driver and workers.
|
||||
# We set CUDA_VISIBLE_DEVICES to ALL GPUs on the node for each worker.
|
||||
# This is needed because:
|
||||
# 1. Ray's compiled DAG needs to find the allocated GPU in
|
||||
# CUDA_VISIBLE_DEVICES.
|
||||
# 2. vLLM's communication layer (NCCL, CustomAllreduce) needs to see
|
||||
# all GPUs for P2P checks and communication setup. Though if it was
|
||||
# just this reason, we could have also just kept the visible devices
|
||||
# unset.
|
||||
# Each worker will use local_rank to index into the visible devices.
|
||||
all_args_to_update_environment_variables = [
|
||||
{
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
map(str, node_gpus[node_id])
|
||||
),
|
||||
}
|
||||
for (node_id, _) in worker_node_and_gpu_ids
|
||||
all_args_to_update_environment_variables: list[dict[str, str]] = [
|
||||
{} for _ in worker_node_and_physical_gpu_ids
|
||||
]
|
||||
|
||||
# Environment variables to copy from driver to workers
|
||||
@@ -336,7 +326,7 @@ class RayDistributedExecutor(Executor):
|
||||
"update_environment_variables", args=(self._get_env_vars_to_be_updated(),)
|
||||
)
|
||||
|
||||
if len(node_gpus) == 1:
|
||||
if len(node_physical_gpu_ids) == 1:
|
||||
# in single node case, we don't need to get the IP address.
|
||||
# the loopback address is sufficient
|
||||
# NOTE: a node may have several IP addresses, one for each
|
||||
@@ -352,10 +342,11 @@ class RayDistributedExecutor(Executor):
|
||||
|
||||
# Initialize the actual workers inside worker wrapper.
|
||||
all_kwargs = []
|
||||
for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
|
||||
for rank, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
|
||||
local_rank = node_workers[node_id].index(rank)
|
||||
kwargs = dict(
|
||||
vllm_config=self.vllm_config,
|
||||
assigned_physical_gpu_ids=sorted(node_physical_gpu_ids[node_id]),
|
||||
local_rank=local_rank,
|
||||
rank=rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
|
||||
@@ -79,24 +79,25 @@ class RayWorkerProc(WorkerProc):
|
||||
1. __init__: lightweight setup, stores init args (no device/model init)
|
||||
2. initialize_worker: called after GPU IDs are discovered, completes
|
||||
the full WorkerProc initialization with the correct local_rank and
|
||||
CUDA_VISIBLE_DEVICES.
|
||||
logical-to-physical GPU mapping.
|
||||
|
||||
CUDA_VISIBLE_DEVICES setup flow:
|
||||
GPU assignment flow:
|
||||
|
||||
1. RayExecutorV2 enables RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES so Ray does
|
||||
not set CUDA_VISIBLE_DEVICES on RayWorkerProc actors at creation time.
|
||||
2. Each actor is scheduled with a placement group and bundle index; Ray resolves
|
||||
the physical GPU ID for that bundle at placement time.
|
||||
3. After placement, the worker discovers that GPU ID and sets
|
||||
CUDA_VISIBLE_DEVICES before finishing WorkerProc initialization.
|
||||
3. After placement, the executor discovers each worker's GPU ID and passes the
|
||||
node's logical-to-physical mapping (assigned_physical_gpu_ids) to
|
||||
initialize_worker(); CUDA_VISIBLE_DEVICES is never modified.
|
||||
|
||||
There is no workaround for this unset-and-reset sequence when the placement group
|
||||
is externally managed: scheduling must complete before CUDA_VISIBLE_DEVICES can
|
||||
match the GPU tied to the worker's bundle.
|
||||
Scheduling must complete before the mapping is known when the placement
|
||||
group is externally managed: only then is the GPU tied to the worker's
|
||||
bundle resolved.
|
||||
|
||||
This sequence allows multiple vLLM instances to coexist on the same node:
|
||||
each instance is unaware which physical devices others hold, and the
|
||||
externally managed placement group avoids CUDA_VISIBLE_DEVICES conflicts
|
||||
externally managed placement group avoids device assignment conflicts
|
||||
by binding workers to specific placement group bundles.
|
||||
"""
|
||||
|
||||
@@ -120,28 +121,33 @@ class RayWorkerProc(WorkerProc):
|
||||
is_driver_worker=is_driver_worker,
|
||||
)
|
||||
|
||||
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, gpu_ids) assigned to this actor by Ray."""
|
||||
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
"""Return (node_id, physical_gpu_ids) assigned to this actor by Ray."""
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
device_key = current_platform.ray_device_key
|
||||
if not device_key:
|
||||
raise RuntimeError(
|
||||
f"current platform {current_platform.device_name} does not support ray."
|
||||
)
|
||||
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, [int(x) for x in gpu_ids]
|
||||
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, [
|
||||
current_platform.device_control_id_to_physical_device_id(str(x))
|
||||
for x in physical_gpu_ids
|
||||
]
|
||||
|
||||
def initialize_worker(
|
||||
self,
|
||||
local_rank: int,
|
||||
env_vars: dict[str, str],
|
||||
driver_env_vars: dict[str, str] | None = None,
|
||||
assigned_physical_gpu_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Complete initialization after GPU assignment is known.
|
||||
|
||||
*driver_env_vars* are applied with ``setdefault`` — they fill
|
||||
in missing vars but never overwrite node-local values.
|
||||
*env_vars* (e.g. CUDA_VISIBLE_DEVICES) always overwrite.
|
||||
*env_vars* always overwrite.
|
||||
*assigned_physical_gpu_ids* maps local_rank to physical CUDA device ID.
|
||||
"""
|
||||
if driver_env_vars:
|
||||
for key, value in driver_env_vars.items():
|
||||
@@ -149,6 +155,13 @@ class RayWorkerProc(WorkerProc):
|
||||
for key, value in env_vars.items():
|
||||
os.environ[key] = value
|
||||
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
vllm_config = self._init_kwargs["vllm_config"]
|
||||
assert isinstance(vllm_config, VllmConfig)
|
||||
vllm_config.parallel_config.assigned_physical_gpu_ids = (
|
||||
assigned_physical_gpu_ids
|
||||
)
|
||||
|
||||
self.local_rank = local_rank
|
||||
super().__init__(
|
||||
local_rank=local_rank,
|
||||
@@ -365,36 +378,48 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
)
|
||||
self.ray_worker_handles.append(handle)
|
||||
|
||||
# Step 6: Discover GPU IDs assigned to each worker via Ray runtime context.
|
||||
worker_node_and_gpu_ids = ray.get(
|
||||
[h.actor.get_node_and_gpu_ids.remote() for h in self.ray_worker_handles]
|
||||
# Step 6: Discover physical GPU IDs assigned to each worker via Ray
|
||||
# runtime context.
|
||||
worker_node_and_physical_gpu_ids = ray.get(
|
||||
[
|
||||
h.actor.get_node_and_physical_gpu_ids.remote()
|
||||
for h in self.ray_worker_handles
|
||||
]
|
||||
)
|
||||
|
||||
node_workers: dict[str, list[int]] = defaultdict(list)
|
||||
node_gpus: dict[str, list[int]] = defaultdict(list)
|
||||
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
|
||||
node_physical_gpu_ids: dict[str, list[int]] = defaultdict(list)
|
||||
for i, (node_id, physical_gpu_ids) in enumerate(
|
||||
worker_node_and_physical_gpu_ids
|
||||
):
|
||||
node_workers[node_id].append(i)
|
||||
node_gpus[node_id].extend(gpu_ids)
|
||||
for node_id, gpu_ids in node_gpus.items():
|
||||
node_gpus[node_id] = sorted(gpu_ids)
|
||||
node_physical_gpu_ids[node_id].extend(physical_gpu_ids)
|
||||
for node_id, physical_gpu_ids in node_physical_gpu_ids.items():
|
||||
node_physical_gpu_ids[node_id] = sorted(physical_gpu_ids)
|
||||
|
||||
# Step 7: Initialize workers with correct local_rank and
|
||||
# CUDA_VISIBLE_DEVICES. Each worker sees all GPUs assigned to
|
||||
# this executor on its node; local_rank indexes into that set.
|
||||
# Step 7: Initialize workers with local logical ranks and the
|
||||
# logical-to-physical GPU mapping discovered from Ray placement.
|
||||
init_worker_refs = []
|
||||
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
|
||||
for i, (node_id, _) in enumerate(worker_node_and_physical_gpu_ids):
|
||||
local_rank = node_workers[node_id].index(i)
|
||||
worker_env_vars = {
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
map(str, node_gpus[node_id])
|
||||
),
|
||||
}
|
||||
assigned_physical_gpu_ids = sorted(node_physical_gpu_ids[node_id])
|
||||
worker_env_vars: dict[str, str] = {}
|
||||
self.ray_worker_handles[i].local_rank = local_rank
|
||||
init_worker_refs.append(
|
||||
self.ray_worker_handles[i].actor.initialize_worker.remote(
|
||||
local_rank, worker_env_vars, self.driver_env_vars
|
||||
local_rank,
|
||||
worker_env_vars,
|
||||
self.driver_env_vars,
|
||||
assigned_physical_gpu_ids=assigned_physical_gpu_ids,
|
||||
)
|
||||
)
|
||||
# Also set on the executor-side config for consistency. The mapping
|
||||
# is per-node, so only do this when all workers share one node.
|
||||
if len(node_physical_gpu_ids) == 1:
|
||||
node_id_0 = worker_node_and_physical_gpu_ids[0][0]
|
||||
self.vllm_config.parallel_config.assigned_physical_gpu_ids = sorted(
|
||||
node_physical_gpu_ids[node_id_0]
|
||||
)
|
||||
ray.get(init_worker_refs)
|
||||
|
||||
# Step 8: Collect response MQ handles
|
||||
|
||||
@@ -93,7 +93,7 @@ try:
|
||||
def get_node_ip(self) -> str:
|
||||
return get_ip()
|
||||
|
||||
def get_node_and_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
def get_node_and_physical_gpu_ids(self) -> tuple[str, list[int]]:
|
||||
node_id = ray.get_runtime_context().get_node_id()
|
||||
device_key = vllm.platforms.current_platform.ray_device_key
|
||||
if not device_key:
|
||||
@@ -101,8 +101,10 @@ try:
|
||||
"current platform %s does not support ray.",
|
||||
vllm.platforms.current_platform.device_name,
|
||||
)
|
||||
gpu_ids = ray.get_runtime_context().get_accelerator_ids()[device_key]
|
||||
return node_id, gpu_ids
|
||||
physical_gpu_ids = ray.get_runtime_context().get_accelerator_ids()[
|
||||
device_key
|
||||
]
|
||||
return node_id, physical_gpu_ids
|
||||
|
||||
def setup_device_if_necessary(self):
|
||||
# TODO(swang): This is needed right now because Ray CG executes
|
||||
|
||||
Reference in New Issue
Block a user