Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -6,6 +6,7 @@ import copy
|
||||
import dataclasses
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import MISSING, asdict, dataclass, fields, is_dataclass
|
||||
@@ -465,6 +466,7 @@ class EngineArgs:
|
||||
numa_bind: bool = ParallelConfig.numa_bind
|
||||
numa_bind_nodes: list[int] | None = ParallelConfig.numa_bind_nodes
|
||||
numa_bind_cpus: list[str] | None = ParallelConfig.numa_bind_cpus
|
||||
device_ids: list[int | str] | None = None
|
||||
tensor_parallel_size: int = ParallelConfig.tensor_parallel_size
|
||||
prefill_context_parallel_size: int = ParallelConfig.prefill_context_parallel_size
|
||||
decode_context_parallel_size: int = ParallelConfig.decode_context_parallel_size
|
||||
@@ -979,6 +981,20 @@ class EngineArgs:
|
||||
parallel_group.add_argument(
|
||||
"--numa-bind-cpus", **parallel_kwargs["numa_bind_cpus"]
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--device-ids",
|
||||
type=lambda s: [
|
||||
int(device_id) if device_id.isdigit() else device_id
|
||||
for device_id in (part.strip() for part in s.split(","))
|
||||
],
|
||||
default=None,
|
||||
help="Comma-separated physical GPU device IDs or UUIDs to use "
|
||||
'(e.g. --device-ids "2,3,5,7"). Avoids setting '
|
||||
"CUDA_VISIBLE_DEVICES, preserving full GPU topology "
|
||||
"visibility for GPU-NIC affinity and DeepGEMM. "
|
||||
"Note: has no effect with Ray executors; use Ray "
|
||||
"placement groups for GPU selection instead.",
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--tensor-parallel-size", "-tp", **parallel_kwargs["tensor_parallel_size"]
|
||||
)
|
||||
@@ -1716,6 +1732,47 @@ class EngineArgs:
|
||||
)
|
||||
return SpeculativeConfig(**self.speculative_config)
|
||||
|
||||
def _resolve_device_ids(self) -> list[int] | None:
|
||||
if not self.device_ids:
|
||||
return None
|
||||
if self.distributed_executor_backend == "ray":
|
||||
logger.warning(
|
||||
"--device-ids has no effect when using the Ray executor. "
|
||||
"Use Ray placement groups for GPU selection instead."
|
||||
)
|
||||
ids = self.device_ids
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError(f"--device-ids must not contain duplicates: {ids}")
|
||||
if all(isinstance(i, str) for i in ids):
|
||||
return [
|
||||
current_platform.device_control_id_to_physical_device_id(i)
|
||||
for i in cast(list[str], ids)
|
||||
]
|
||||
if any(isinstance(i, str) for i in ids):
|
||||
raise ValueError("--device-ids must not mix integer IDs and UUIDs")
|
||||
int_ids = cast(list[int], ids)
|
||||
# Compose with CUDA_VISIBLE_DEVICES: if CVD is set, treat
|
||||
# --device-ids values as indices into the CVD-visible set.
|
||||
cvd = getattr(
|
||||
envs,
|
||||
current_platform.device_control_env_var,
|
||||
os.environ.get(current_platform.device_control_env_var),
|
||||
)
|
||||
if cvd:
|
||||
cvd_ids = [
|
||||
current_platform.device_control_id_to_physical_device_id(x)
|
||||
for x in cvd.split(",")
|
||||
]
|
||||
for i in int_ids:
|
||||
if i >= len(cvd_ids):
|
||||
raise ValueError(
|
||||
f"--device-ids index {i} is out of range for "
|
||||
f"{current_platform.device_control_env_var}"
|
||||
f"={cvd} ({len(cvd_ids)} devices visible)"
|
||||
)
|
||||
return [cvd_ids[i] for i in int_ids]
|
||||
return int_ids
|
||||
|
||||
def create_diffusion_config(self) -> DiffusionConfig | None:
|
||||
if self.diffusion_config is None:
|
||||
return None
|
||||
@@ -2029,6 +2086,7 @@ class EngineArgs:
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
_api_process_count=self._api_process_count,
|
||||
_api_process_rank=self._api_process_rank,
|
||||
assigned_physical_gpu_ids=self._resolve_device_ids(),
|
||||
numa_bind=self.numa_bind,
|
||||
numa_bind_nodes=self.numa_bind_nodes,
|
||||
numa_bind_cpus=self.numa_bind_cpus,
|
||||
|
||||
Reference in New Issue
Block a user