Stop setting CUDA_VISIBLE_DEVICES internally in vLLM, add device_ids arg (#45026)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <kouroshHakha@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kourosh hakhamaneshi
parent
e9de72fe6c
commit
ebfbcfe46a
@@ -482,10 +482,11 @@ def _init_lmcache_engine(
|
||||
)
|
||||
|
||||
# Change current device.
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
local_rank = parallel_config.rank % num_gpus
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
device_index = get_world_group().device_index
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
device = torch.device(f"cuda:{device_index}")
|
||||
metadata = LMCacheEngineMetadata(
|
||||
model_config.model,
|
||||
parallel_config.world_size,
|
||||
|
||||
Reference in New Issue
Block a user