[Hardware][Intel] OpenVINO vLLM backend (#5379)

This commit is contained in:
Ilya Lavrenov
2024-06-28 13:50:16 +00:00
committed by GitHub
parent 5932634409
commit 57f09a419c
22 changed files with 1393 additions and 23 deletions
+5 -3
View File
@@ -14,8 +14,8 @@ from vllm.model_executor.models import ModelRegistry
from vllm.tracing import is_otel_installed
from vllm.transformers_utils.config import get_config, get_hf_text_config
from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
is_hip, is_neuron, is_tpu, is_xpu, print_warning_once,
update_environment_variables)
is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
print_warning_once, update_environment_variables)
if TYPE_CHECKING:
from ray.util.placement_group import PlacementGroup
@@ -781,6 +781,8 @@ class DeviceConfig:
# Automated device type detection
if is_neuron():
self.device_type = "neuron"
elif is_openvino():
self.device_type = "openvino"
elif is_tpu():
self.device_type = "tpu"
elif is_cpu():
@@ -796,7 +798,7 @@ class DeviceConfig:
self.device_type = device
# Some device types require processing inputs on CPU
if self.device_type in ["neuron"]:
if self.device_type in ["neuron", "openvino"]:
self.device = torch.device("cpu")
elif self.device_type in ["tpu"]:
self.device = None