[CPU] V1 support for the CPU backend (#16441)
This commit is contained in:
+57
-10
@@ -57,7 +57,10 @@ class CpuPlatform(Platform):
|
||||
logger.info("Using CPU MLA backend.")
|
||||
return "vllm.attention.backends.cpu_mla.CPUMLABackend"
|
||||
logger.info("Using Torch SDPA backend.")
|
||||
return "vllm.attention.backends.torch_sdpa.TorchSDPABackend"
|
||||
if use_v1:
|
||||
return "vllm.v1.attention.backends.cpu_attn.TorchSDPABackend"
|
||||
else:
|
||||
return "vllm.attention.backends.torch_sdpa.TorchSDPABackend"
|
||||
|
||||
@classmethod
|
||||
def get_device_total_memory(cls, device_id: int = 0) -> int:
|
||||
@@ -81,6 +84,8 @@ class CpuPlatform(Platform):
|
||||
if not model_config.enforce_eager:
|
||||
model_config.enforce_eager = True
|
||||
|
||||
model_config.disable_cascade_attn = True
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
ipex_available = find_spec("intel_extension_for_pytorch") is not None
|
||||
@@ -128,7 +133,8 @@ class CpuPlatform(Platform):
|
||||
f" {kv_cache_space}, expect a positive integer value.")
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if (parallel_config.distributed_executor_backend is not None
|
||||
if (parallel_config.world_size > 1
|
||||
and parallel_config.distributed_executor_backend is not None
|
||||
and parallel_config.distributed_executor_backend != "mp"):
|
||||
logger.warning(("%s is not supported on CPU, fallback to mp "
|
||||
"distributed executor backend."),
|
||||
@@ -141,7 +147,38 @@ class CpuPlatform(Platform):
|
||||
parallel_config.sd_worker_cls = \
|
||||
"vllm.worker.cpu_worker.CPUWorker"
|
||||
else:
|
||||
parallel_config.worker_cls = "vllm.worker.cpu_worker.CPUWorker"
|
||||
if envs.VLLM_USE_V1:
|
||||
parallel_config.worker_cls = \
|
||||
"vllm.v1.worker.cpu_worker.CPUWorker"
|
||||
else:
|
||||
parallel_config.worker_cls = \
|
||||
"vllm.worker.cpu_worker.CPUWorker"
|
||||
|
||||
# Note: workaround for v1 gpu_model_runner
|
||||
from vllm.config import CompilationLevel
|
||||
vllm_config.compilation_config.cudagraph_capture_sizes = []
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if (envs.VLLM_USE_V1 and vllm_config.compilation_config.level
|
||||
== CompilationLevel.PIECEWISE):
|
||||
compilation_config.level = CompilationLevel.DYNAMO_ONCE
|
||||
compilation_config.backend = "eager"
|
||||
compilation_config.custom_ops += ["none"]
|
||||
compilation_config.inductor_compile_config.update({
|
||||
"dce":
|
||||
True,
|
||||
"size_asserts":
|
||||
False,
|
||||
"nan_asserts":
|
||||
False,
|
||||
"memory_planning":
|
||||
True,
|
||||
"epilogue_fusion":
|
||||
True,
|
||||
})
|
||||
|
||||
if vllm_config.lora_config is not None:
|
||||
compilation_config.level = CompilationLevel.NO_COMPILATION
|
||||
|
||||
assert vllm_config.device_config.device_type == "cpu"
|
||||
|
||||
@@ -149,6 +186,12 @@ class CpuPlatform(Platform):
|
||||
# Environment variables for CPU executor
|
||||
#
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
# Note: to avoid the error 'nthreads cannot be larger than environment
|
||||
# variable "NUMEXPR_MAX_THREADS" (64)'.
|
||||
os.environ["NUMEXPR_MAX_THREADS"] = str(len(os.sched_getaffinity(0)))
|
||||
|
||||
# Set default threads num for OpenMP parallel
|
||||
os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())
|
||||
|
||||
@@ -171,13 +214,6 @@ class CpuPlatform(Platform):
|
||||
# To hint IPEX uses shared memory based AllReduce
|
||||
os.environ["LOCAL_WORLD_SIZE"] = str(
|
||||
vllm_config.parallel_config.tensor_parallel_size)
|
||||
if sys.platform == "darwin" and \
|
||||
envs.VLLM_WORKER_MULTIPROC_METHOD == "fork":
|
||||
if os.environ.get('VLLM_WORKER_MULTIPROC_METHOD', None) is None:
|
||||
logger.warning(
|
||||
"Default to spawn method on MacOS. If this is not desired,"
|
||||
" set VLLM_WORKER_MULTIPROC_METHOD to fork explicitly.")
|
||||
os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
|
||||
|
||||
if vllm_config.model_config and vllm_config.model_config.use_mla:
|
||||
logger.info(
|
||||
@@ -204,3 +240,14 @@ class CpuPlatform(Platform):
|
||||
Get device specific communicator class for distributed communication.
|
||||
"""
|
||||
return "vllm.distributed.device_communicators.cpu_communicator.CpuCommunicator" # noqa
|
||||
|
||||
@classmethod
|
||||
def supports_structured_output(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_v1(cls, model_config) -> bool:
|
||||
"""Returns whether the current platform can support v1 for the supplied
|
||||
model configuration.
|
||||
"""
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user