Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -270,19 +270,47 @@ class Worker(WorkerBase):
|
||||
|
||||
# DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK
|
||||
self.local_rank += dp_local_rank * tp_pp_world_size
|
||||
|
||||
# Publish the logical-to-physical mapping for topology queries
|
||||
# such as NIC affinity and P2P checks.
|
||||
assigned_physical_gpu_ids = parallel_config.assigned_physical_gpu_ids
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
from vllm.platforms.interface import set_assigned_physical_gpu_ids
|
||||
|
||||
set_assigned_physical_gpu_ids(assigned_physical_gpu_ids)
|
||||
assert self.local_rank < len(assigned_physical_gpu_ids), (
|
||||
f"local_rank {self.local_rank} is out of bounds for "
|
||||
f"assigned_physical_gpu_ids {assigned_physical_gpu_ids}"
|
||||
)
|
||||
# NOTE(patch pr45026): local_world_size is derived from
|
||||
# parallel_config.nnodes, which is only set for the "mp"
|
||||
# multi-node backend. With the "ray"/"external_launcher"
|
||||
# backends nnodes stays 1, so local_world_size collapses to
|
||||
# the full world_size and this check wrongly fires on
|
||||
# cross-node deployments. assigned_physical_gpu_ids is already
|
||||
# per-node and the local_rank bound above fully validates the
|
||||
# mapping for these backends, so skip the check for them.
|
||||
if parallel_config.distributed_executor_backend not in (
|
||||
"ray",
|
||||
"external_launcher",
|
||||
):
|
||||
assert self.parallel_config.local_world_size <= len(
|
||||
assigned_physical_gpu_ids
|
||||
), (
|
||||
f"local_world_size ({self.parallel_config.local_world_size})"
|
||||
" exceeds assigned_physical_gpu_ids count "
|
||||
f"({len(assigned_physical_gpu_ids)})"
|
||||
)
|
||||
else:
|
||||
assert self.local_rank < torch.accelerator.device_count(), (
|
||||
f"DP adjusted local rank {self.local_rank} is out of bounds. "
|
||||
)
|
||||
visible_device_count = (
|
||||
torch.accelerator.device_count() if torch.cuda.is_available() else 0
|
||||
)
|
||||
assert self.parallel_config.local_world_size <= visible_device_count, (
|
||||
f"local_world_size ({self.parallel_config.local_world_size}) must "
|
||||
f"be less than or equal to the number of visible devices "
|
||||
f"({visible_device_count})."
|
||||
f"DP adjusted local rank {self.local_rank} is out of "
|
||||
f"bounds for {torch.accelerator.device_count()} devices."
|
||||
)
|
||||
|
||||
self.device = torch.device(f"cuda:{self.local_rank}")
|
||||
visible_device_index = (
|
||||
current_platform.logical_device_id_to_visible_device_id(self.local_rank)
|
||||
)
|
||||
self.device = torch.device(f"cuda:{visible_device_index}")
|
||||
torch.accelerator.set_device_index(self.device)
|
||||
|
||||
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
||||
|
||||
Reference in New Issue
Block a user