platforms: add uses_cpu_device() hook to Platform for DeviceConfig (#42313)
Signed-off-by: Viktor Pus <viktorpus@tenstorrent.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
379f0ec369
commit
0ce6613b9c
@@ -65,8 +65,13 @@ class DeviceConfig:
|
||||
elif isinstance(self.device, torch.device):
|
||||
self.device_type = self.device.type
|
||||
|
||||
# Some device types require processing inputs on CPU
|
||||
if self.device_type in ["tpu"]:
|
||||
# Some platforms require processing inputs on CPU.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if (
|
||||
current_platform.uses_host_device_handling()
|
||||
and self.device_type == current_platform.device_type
|
||||
):
|
||||
self.device = None
|
||||
else:
|
||||
# Set device with device type
|
||||
|
||||
@@ -172,6 +172,10 @@ class Platform:
|
||||
def is_cpu(self) -> bool:
|
||||
return self._enum == PlatformEnum.CPU
|
||||
|
||||
def uses_host_device_handling(self) -> bool:
|
||||
"""Whether vLLM should leave DeviceConfig.device unset."""
|
||||
return self.is_tpu()
|
||||
|
||||
def is_zen_cpu(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user