Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -392,6 +392,14 @@ class GroupCoordinator:
|
||||
|
||||
self.rank = torch.distributed.get_rank()
|
||||
self.local_rank = local_rank
|
||||
self.device_index: int
|
||||
if _WORLD is not None:
|
||||
self.device_index = _WORLD.device_index
|
||||
else:
|
||||
assert local_rank >= 0, (
|
||||
"local_rank must be provided when creating the world group"
|
||||
)
|
||||
self.device_index = local_rank
|
||||
|
||||
self_device_group = None
|
||||
self_cpu_group = None
|
||||
@@ -442,11 +450,18 @@ class GroupCoordinator:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
self.device = torch.device(f"cuda:{local_rank}")
|
||||
visible_device_index = (
|
||||
current_platform.logical_device_id_to_visible_device_id(
|
||||
self.device_index
|
||||
)
|
||||
)
|
||||
self.device = torch.device(f"cuda:{visible_device_index}")
|
||||
elif current_platform.is_xpu():
|
||||
self.device = torch.device(f"xpu:{local_rank}")
|
||||
self.device = torch.device(f"xpu:{self.device_index}")
|
||||
elif current_platform.is_out_of_tree():
|
||||
self.device = torch.device(f"{current_platform.device_name}:{local_rank}")
|
||||
self.device = torch.device(
|
||||
f"{current_platform.device_name}:{self.device_index}"
|
||||
)
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
@@ -1438,7 +1453,12 @@ def _init_process_group_for_split_group(
|
||||
"""
|
||||
if torch.accelerator.is_available() and backend != "gloo":
|
||||
init_backend = "cpu:gloo,cuda:nccl"
|
||||
device_id: torch.device | None = torch.device(f"cuda:{local_rank}")
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
visible_device_index = current_platform.logical_device_id_to_visible_device_id(
|
||||
local_rank
|
||||
)
|
||||
device_id: torch.device | None = torch.device(f"cuda:{visible_device_index}")
|
||||
else:
|
||||
init_backend = "gloo"
|
||||
device_id = None
|
||||
|
||||
Reference in New Issue
Block a user