Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -704,7 +704,14 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
self.num_experts = num_experts
|
||||
|
||||
self.cleanup()
|
||||
gpus_per_node = torch.accelerator.device_count()
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
assigned_physical_gpu_ids = get_assigned_physical_gpu_ids()
|
||||
gpus_per_node = (
|
||||
len(assigned_physical_gpu_ids)
|
||||
if assigned_physical_gpu_ids is not None
|
||||
else torch.accelerator.device_count()
|
||||
)
|
||||
logger.debug(
|
||||
"Making One-sided NVLink mapping: rank=%d, world size=%d",
|
||||
self.rank,
|
||||
|
||||
@@ -320,13 +320,21 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
|
||||
is_distributed = dist.is_initialized()
|
||||
|
||||
num_dev = current_platform.device_count()
|
||||
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
|
||||
if cuda_visible_devices is None:
|
||||
cuda_visible_devices = ",".join(str(i) for i in range(num_dev))
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
assigned_physical_gpu_ids = get_assigned_physical_gpu_ids()
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
# Key by the ordered list: the cache stores directed local-index
|
||||
# pairs, so permutations of the same set are distinct mappings.
|
||||
cache_key = ",".join(str(i) for i in assigned_physical_gpu_ids)
|
||||
num_dev = len(assigned_physical_gpu_ids)
|
||||
else:
|
||||
num_dev = current_platform.device_count()
|
||||
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
|
||||
cache_key = cuda_visible_devices or ",".join(str(i) for i in range(num_dev))
|
||||
|
||||
path = os.path.join(
|
||||
envs.VLLM_CACHE_ROOT, f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
|
||||
envs.VLLM_CACHE_ROOT, f"gpu_p2p_access_cache_for_{cache_key}.json"
|
||||
)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
@@ -338,7 +346,15 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
# enter this block to calculate the cache
|
||||
logger.info("generating GPU P2P access cache in %s", path)
|
||||
cache: dict[str, bool] = {}
|
||||
ids = list(range(num_dev))
|
||||
# The probe subprocesses inherit this process's device-control env
|
||||
# var, so they must be given visible ordinals, not physical IDs.
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
ids = [
|
||||
current_platform.logical_device_id_to_visible_device_id(local)
|
||||
for local in range(num_dev)
|
||||
]
|
||||
else:
|
||||
ids = list(range(num_dev))
|
||||
# batch of all pairs of GPUs
|
||||
batch_src, batch_tgt = zip(*list(product(ids, ids)))
|
||||
# NOTE: we use `subprocess` rather than `multiprocessing` here
|
||||
@@ -368,8 +384,11 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
) from e
|
||||
with open(output_file.name, "rb") as f:
|
||||
result = pickle.load(f)
|
||||
# Cache entries must be keyed by local indices (0..N-1) because
|
||||
# gpu_p2p_access_check() is called with local ranks.
|
||||
id_to_local = {device_id: local for local, device_id in enumerate(ids)}
|
||||
for _i, _j, r in zip(batch_src, batch_tgt, result):
|
||||
cache[f"{_i}->{_j}"] = r
|
||||
cache[f"{id_to_local[_i]}->{id_to_local[_j]}"] = r
|
||||
with open(path, "w") as f:
|
||||
json.dump(cache, f, indent=4)
|
||||
if is_distributed:
|
||||
|
||||
@@ -34,7 +34,12 @@ def _can_p2p(rank: int, world_size: int) -> bool:
|
||||
continue
|
||||
if envs.VLLM_SKIP_P2P_CHECK:
|
||||
logger.debug("Skipping P2P check and trusting the driver's P2P report.")
|
||||
return torch.cuda.can_device_access_peer(rank, i)
|
||||
# can_device_access_peer takes visible device ordinals, while
|
||||
# rank and i are logical local IDs.
|
||||
return torch.cuda.can_device_access_peer(
|
||||
current_platform.logical_device_id_to_visible_device_id(rank),
|
||||
current_platform.logical_device_id_to_visible_device_id(i),
|
||||
)
|
||||
if not gpu_p2p_access_check(rank, i):
|
||||
return False
|
||||
return True
|
||||
@@ -126,13 +131,10 @@ class CustomAllreduce:
|
||||
CUSTOM_ALL_REDUCE_MAX_SIZES[device_capability_str][world_size],
|
||||
max_size,
|
||||
)
|
||||
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
|
||||
if cuda_visible_devices:
|
||||
device_ids = list(map(int, cuda_visible_devices.split(",")))
|
||||
else:
|
||||
device_ids = list(range(current_platform.device_count()))
|
||||
|
||||
physical_device_id = device_ids[device.index]
|
||||
# device.index is a visible ordinal, not a logical local ID.
|
||||
physical_device_id = current_platform.visible_device_id_to_physical_device_id(
|
||||
device.index
|
||||
)
|
||||
tensor = torch.tensor([physical_device_id], dtype=torch.int, device="cpu")
|
||||
gather_list = [
|
||||
torch.tensor([0], dtype=torch.int, device="cpu") for _ in range(world_size)
|
||||
|
||||
@@ -129,12 +129,10 @@ class QuickAllReduce:
|
||||
assert isinstance(device, torch.device)
|
||||
self.device = device
|
||||
|
||||
cuda_visible_devices = envs.CUDA_VISIBLE_DEVICES
|
||||
if cuda_visible_devices:
|
||||
device_ids = list(map(int, cuda_visible_devices.split(",")))
|
||||
else:
|
||||
device_ids = list(range(current_platform.device_count()))
|
||||
physical_device_id = device_ids[device.index]
|
||||
# device.index is a visible ordinal, not a logical local ID.
|
||||
physical_device_id = current_platform.visible_device_id_to_physical_device_id(
|
||||
device.index
|
||||
)
|
||||
tensor = torch.tensor([physical_device_id], dtype=torch.int, device="cpu")
|
||||
gather_list = [
|
||||
torch.tensor([0], dtype=torch.int, device="cpu")
|
||||
|
||||
@@ -840,7 +840,13 @@ class MessageQueue:
|
||||
The MessageQueue instance for the calling process,
|
||||
and a list of handles (only non-empty for the reader process).
|
||||
"""
|
||||
local_size = current_platform.device_count()
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
assigned_physical_gpu_ids = get_assigned_physical_gpu_ids()
|
||||
if assigned_physical_gpu_ids is not None:
|
||||
local_size = len(assigned_physical_gpu_ids)
|
||||
else:
|
||||
local_size = current_platform.device_count()
|
||||
rank = dist.get_rank()
|
||||
same_node = rank // local_size == reader_rank // local_size
|
||||
buffer_io = MessageQueue(
|
||||
|
||||
@@ -482,10 +482,11 @@ def _init_lmcache_engine(
|
||||
)
|
||||
|
||||
# Change current device.
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
local_rank = parallel_config.rank % num_gpus
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
device_index = get_world_group().device_index
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
device = torch.device(f"cuda:{device_index}")
|
||||
metadata = LMCacheEngineMetadata(
|
||||
model_config.model,
|
||||
parallel_config.world_size,
|
||||
|
||||
@@ -392,6 +392,14 @@ class GroupCoordinator:
|
||||
|
||||
self.rank = torch.distributed.get_rank()
|
||||
self.local_rank = local_rank
|
||||
self.device_index: int
|
||||
if _WORLD is not None:
|
||||
self.device_index = _WORLD.device_index
|
||||
else:
|
||||
assert local_rank >= 0, (
|
||||
"local_rank must be provided when creating the world group"
|
||||
)
|
||||
self.device_index = local_rank
|
||||
|
||||
self_device_group = None
|
||||
self_cpu_group = None
|
||||
@@ -442,11 +450,18 @@ class GroupCoordinator:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
self.device = torch.device(f"cuda:{local_rank}")
|
||||
visible_device_index = (
|
||||
current_platform.logical_device_id_to_visible_device_id(
|
||||
self.device_index
|
||||
)
|
||||
)
|
||||
self.device = torch.device(f"cuda:{visible_device_index}")
|
||||
elif current_platform.is_xpu():
|
||||
self.device = torch.device(f"xpu:{local_rank}")
|
||||
self.device = torch.device(f"xpu:{self.device_index}")
|
||||
elif current_platform.is_out_of_tree():
|
||||
self.device = torch.device(f"{current_platform.device_name}:{local_rank}")
|
||||
self.device = torch.device(
|
||||
f"{current_platform.device_name}:{self.device_index}"
|
||||
)
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
@@ -1438,7 +1453,12 @@ def _init_process_group_for_split_group(
|
||||
"""
|
||||
if torch.accelerator.is_available() and backend != "gloo":
|
||||
init_backend = "cpu:gloo,cuda:nccl"
|
||||
device_id: torch.device | None = torch.device(f"cuda:{local_rank}")
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
visible_device_index = current_platform.logical_device_id_to_visible_device_id(
|
||||
local_rank
|
||||
)
|
||||
device_id: torch.device | None = torch.device(f"cuda:{visible_device_index}")
|
||||
else:
|
||||
init_backend = "gloo"
|
||||
device_id = None
|
||||
|
||||
@@ -86,6 +86,15 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
|
||||
self.rank = global_rank
|
||||
self.local_rank = local_rank
|
||||
from vllm.distributed.parallel_state import _WORLD
|
||||
|
||||
if _WORLD is not None:
|
||||
self.device_index = _WORLD.device_index
|
||||
else:
|
||||
assert local_rank >= 0, (
|
||||
"local_rank must be provided when creating the world group"
|
||||
)
|
||||
self.device_index = local_rank
|
||||
|
||||
self_device_group = None
|
||||
self_cpu_group = None
|
||||
@@ -152,11 +161,18 @@ class StatelessGroupCoordinator(GroupCoordinator):
|
||||
self.tcp_store_group = self_tcp_store_group
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
self.device = torch.device(f"cuda:{local_rank}")
|
||||
visible_device_index = (
|
||||
current_platform.logical_device_id_to_visible_device_id(
|
||||
self.device_index
|
||||
)
|
||||
)
|
||||
self.device = torch.device(f"cuda:{visible_device_index}")
|
||||
elif current_platform.is_xpu():
|
||||
self.device = torch.device(f"xpu:{local_rank}")
|
||||
self.device = torch.device(f"xpu:{self.device_index}")
|
||||
elif current_platform.is_out_of_tree():
|
||||
self.device = torch.device(f"{current_platform.device_name}:{local_rank}")
|
||||
self.device = torch.device(
|
||||
f"{current_platform.device_name}:{self.device_index}"
|
||||
)
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user