diff --git a/vllm/config/device.py b/vllm/config/device.py index c20e4d0f28..2d46dcdb11 100644 --- a/vllm/config/device.py +++ b/vllm/config/device.py @@ -65,8 +65,13 @@ class DeviceConfig: elif isinstance(self.device, torch.device): self.device_type = self.device.type - # Some device types require processing inputs on CPU - if self.device_type in ["tpu"]: + # Some platforms require processing inputs on CPU. + from vllm.platforms import current_platform + + if ( + current_platform.uses_host_device_handling() + and self.device_type == current_platform.device_type + ): self.device = None else: # Set device with device type diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 80952ced73..9a93ef9f82 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -172,6 +172,10 @@ class Platform: def is_cpu(self) -> bool: return self._enum == PlatformEnum.CPU + def uses_host_device_handling(self) -> bool: + """Whether vLLM should leave DeviceConfig.device unset.""" + return self.is_tpu() + def is_zen_cpu(self) -> bool: return False