diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index a76331f4d5..ef2b4ba073 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -97,7 +97,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'cd tests && + 'cd tests && pip install dpctl && pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py && pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index ffc9a5de7e..62b6ef880e 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -2452,13 +2452,10 @@ class EngineCoreActorMixin: def _set_visible_devices(self, vllm_config: VllmConfig, local_dp_rank: int): from vllm.platforms import current_platform - if current_platform.is_xpu(): - pass - else: - device_control_env_var = current_platform.device_control_env_var - self._set_assigned_physical_gpu_ids( - vllm_config, local_dp_rank, device_control_env_var - ) + device_control_env_var = current_platform.device_control_env_var + self._set_assigned_physical_gpu_ids( + vllm_config, local_dp_rank, device_control_env_var + ) def _set_assigned_physical_gpu_ids( self, diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index 68d6bde3a0..5c5d125f4b 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -115,27 +115,55 @@ class XPUWorker(Worker): self.local_rank += dp_local_rank * replica_world_size - assert self.local_rank < visible_device_count, ( - f"DP adjusted local rank {self.local_rank} is out of bounds. " - ) - assert parallel_config.local_world_size <= visible_device_count, ( - f"local_world_size ({parallel_config.local_world_size}) must " - f"be less than or equal to the number of visible devices " - f"({visible_device_count})." - ) - device = self.device_config.device if ( isinstance(device, torch.device) and device.type == "xpu" and current_platform.is_xpu() ): - self.device = torch.device(f"xpu:{self.local_rank}") + assigned_physical_gpu_ids = self.parallel_config.assigned_physical_gpu_ids + if assigned_physical_gpu_ids is not None: + from vllm.platforms.interface import set_assigned_physical_gpu_ids + + set_assigned_physical_gpu_ids(assigned_physical_gpu_ids) + assert self.local_rank < len(assigned_physical_gpu_ids), ( + f"local_rank {self.local_rank} is out of bounds for " + f"assigned_physical_gpu_ids {assigned_physical_gpu_ids}" + ) + # NOTE: local_world_size is derived from parallel_config.nnodes, + # which is only set for the "mp" multi-node backend. With the + # "ray"/"external_launcher" backends nnodes stays 1, so + # local_world_size collapses to the full world_size and this + # check wrongly fires on cross-node deployments. + # assigned_physical_gpu_ids is already per-node and the + # local_rank bound above fully validates the mapping for + # these backends, so skip the check for them. + if parallel_config.distributed_executor_backend not in ( + "ray", + "external_launcher", + ): + assert parallel_config.local_world_size <= len( + assigned_physical_gpu_ids + ), ( + f"local_world_size ({parallel_config.local_world_size})" + " exceeds assigned_physical_gpu_ids count " + f"({len(assigned_physical_gpu_ids)})" + ) + else: + assert self.local_rank < torch.accelerator.device_count(), ( + f"DP adjusted local rank {self.local_rank} is out of " + f"bounds for {torch.accelerator.device_count()} devices." + ) + + visible_device_index = ( + current_platform.logical_device_id_to_visible_device_id(self.local_rank) + ) + self.device = torch.device(f"xpu:{visible_device_index}") torch.accelerator.set_device_index(self.device) current_platform.check_if_supports_dtype(self.model_config.dtype) torch.accelerator.empty_cache() self.init_gpu_memory = torch.xpu.get_device_properties( - self.local_rank + visible_device_index ).total_memory else: raise RuntimeError(f"Unsupported device type: {self.device_config.device}")