[Hardware][Intel] Add CPU inference backend (#3634)

Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Co-authored-by: Yuan Zhou <yuan.zhou@intel.com>
This commit is contained in:
bigPYJ1151
2024-04-01 22:07:30 -07:00
committed by GitHub
co-authored by Kunshang Ji Yuan Zhou
parent eb69d68804
commit 0e3f06fe9c
24 changed files with 2747 additions and 5 deletions
+4 -1
View File
@@ -10,7 +10,8 @@ from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.transformers_utils.config import get_config, get_hf_text_config
from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
is_neuron)
if TYPE_CHECKING:
from ray.util.placement_group import PlacementGroup
@@ -598,6 +599,8 @@ class DeviceConfig:
# Automated device type detection
if is_neuron():
self.device_type = "neuron"
elif is_cpu():
self.device_type = "cpu"
else:
# We don't call torch.cuda.is_available() here to
# avoid initializing CUDA before workers are forked