[XPU] Honor device_ids for worker device placement (#56015)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
c16bb6068f
commit
622934e61f
@@ -97,7 +97,7 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- >-
|
- >-
|
||||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||||
'cd tests &&
|
'cd tests && pip install dpctl &&
|
||||||
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
|
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
|
||||||
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
|
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
|
||||||
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
|
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
|
||||||
|
|||||||
@@ -2452,13 +2452,10 @@ class EngineCoreActorMixin:
|
|||||||
def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int):
|
def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int):
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
if current_platform.is_xpu():
|
device_control_env_var = current_platform.device_control_env_var
|
||||||
pass
|
self._set_assigned_physical_gpu_ids(
|
||||||
else:
|
vllm_config, local_dp_rank, device_control_env_var
|
||||||
device_control_env_var = current_platform.device_control_env_var
|
)
|
||||||
self._set_assigned_physical_gpu_ids(
|
|
||||||
vllm_config, local_dp_rank, device_control_env_var
|
|
||||||
)
|
|
||||||
|
|
||||||
def _set_assigned_physical_gpu_ids(
|
def _set_assigned_physical_gpu_ids(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -115,27 +115,55 @@ class XPUWorker(Worker):
|
|||||||
|
|
||||||
self.local_rank += dp_local_rank * replica_world_size
|
self.local_rank += dp_local_rank * replica_world_size
|
||||||
|
|
||||||
assert self.local_rank < visible_device_count, (
|
|
||||||
f"DP adjusted local rank {self.local_rank} is out of bounds. "
|
|
||||||
)
|
|
||||||
assert parallel_config.local_world_size <= visible_device_count, (
|
|
||||||
f"local_world_size ({parallel_config.local_world_size}) must "
|
|
||||||
f"be less than or equal to the number of visible devices "
|
|
||||||
f"({visible_device_count})."
|
|
||||||
)
|
|
||||||
|
|
||||||
device = self.device_config.device
|
device = self.device_config.device
|
||||||
if (
|
if (
|
||||||
isinstance(device, torch.device)
|
isinstance(device, torch.device)
|
||||||
and device.type == "xpu"
|
and device.type == "xpu"
|
||||||
and current_platform.is_xpu()
|
and current_platform.is_xpu()
|
||||||
):
|
):
|
||||||
self.device = torch.device(f"xpu:{self.local_rank}")
|
assigned_physical_gpu_ids = self.parallel_config.assigned_physical_gpu_ids
|
||||||
|
if assigned_physical_gpu_ids is not None:
|
||||||
|
from vllm.platforms.interface import set_assigned_physical_gpu_ids
|
||||||
|
|
||||||
|
set_assigned_physical_gpu_ids(assigned_physical_gpu_ids)
|
||||||
|
assert self.local_rank < len(assigned_physical_gpu_ids), (
|
||||||
|
f"local_rank {self.local_rank} is out of bounds for "
|
||||||
|
f"assigned_physical_gpu_ids {assigned_physical_gpu_ids}"
|
||||||
|
)
|
||||||
|
# NOTE: local_world_size is derived from parallel_config.nnodes,
|
||||||
|
# which is only set for the "mp" multi-node backend. With the
|
||||||
|
# "ray"/"external_launcher" backends nnodes stays 1, so
|
||||||
|
# local_world_size collapses to the full world_size and this
|
||||||
|
# check wrongly fires on cross-node deployments.
|
||||||
|
# assigned_physical_gpu_ids is already per-node and the
|
||||||
|
# local_rank bound above fully validates the mapping for
|
||||||
|
# these backends, so skip the check for them.
|
||||||
|
if parallel_config.distributed_executor_backend not in (
|
||||||
|
"ray",
|
||||||
|
"external_launcher",
|
||||||
|
):
|
||||||
|
assert parallel_config.local_world_size <= len(
|
||||||
|
assigned_physical_gpu_ids
|
||||||
|
), (
|
||||||
|
f"local_world_size ({parallel_config.local_world_size})"
|
||||||
|
" exceeds assigned_physical_gpu_ids count "
|
||||||
|
f"({len(assigned_physical_gpu_ids)})"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert self.local_rank < torch.accelerator.device_count(), (
|
||||||
|
f"DP adjusted local rank {self.local_rank} is out of "
|
||||||
|
f"bounds for {torch.accelerator.device_count()} devices."
|
||||||
|
)
|
||||||
|
|
||||||
|
visible_device_index = (
|
||||||
|
current_platform.logical_device_id_to_visible_device_id(self.local_rank)
|
||||||
|
)
|
||||||
|
self.device = torch.device(f"xpu:{visible_device_index}")
|
||||||
torch.accelerator.set_device_index(self.device)
|
torch.accelerator.set_device_index(self.device)
|
||||||
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
||||||
torch.accelerator.empty_cache()
|
torch.accelerator.empty_cache()
|
||||||
self.init_gpu_memory = torch.xpu.get_device_properties(
|
self.init_gpu_memory = torch.xpu.get_device_properties(
|
||||||
self.local_rank
|
visible_device_index
|
||||||
).total_memory
|
).total_memory
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(f"Unsupported device type: {self.device_config.device}")
|
raise RuntimeError(f"Unsupported device type: {self.device_config.device}")
|
||||||
|
|||||||
Reference in New Issue
Block a user