[CPU] Enable data parallel for CPU backend (#23903)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -69,6 +69,7 @@ class CpuPlatform(Platform):
|
||||
device_type: str = "cpu"
|
||||
dispatch_key: str = "CPU"
|
||||
dist_backend: str = "gloo"
|
||||
device_control_env_var = "CPU_VISIBLE_MEMORY_NODES"
|
||||
|
||||
@property
|
||||
def supported_dtypes(self) -> list[torch.dtype]:
|
||||
@@ -297,6 +298,13 @@ class CpuPlatform(Platform):
|
||||
allowed_numa_nodes.add(x.numa_node) # type: ignore
|
||||
allowed_numa_nodes_list = sorted(allowed_numa_nodes)
|
||||
|
||||
env_key = CpuPlatform.device_control_env_var
|
||||
if (env_key in os.environ and os.environ[env_key] != ""):
|
||||
visible_nodes = [int(s) for s in os.environ[env_key].split(',')]
|
||||
allowed_numa_nodes_list = [
|
||||
x for x in visible_nodes if x in allowed_cpu_id_list
|
||||
]
|
||||
|
||||
return allowed_numa_nodes_list, logical_cpu_list
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -113,6 +113,11 @@ class CPUModelRunner(GPUModelRunner):
|
||||
def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
|
||||
return sampled_token_ids.tolist()
|
||||
|
||||
def get_dp_padding(self,
|
||||
num_tokens: int) -> tuple[int, Optional[torch.Tensor]]:
|
||||
# Note: For CPU backend, dp padding is not required for now.
|
||||
return 0, None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _torch_cuda_wrapper():
|
||||
|
||||
@@ -55,7 +55,14 @@ class CPUWorker(Worker):
|
||||
else:
|
||||
self.local_omp_cpuid = "all"
|
||||
else:
|
||||
self.local_omp_cpuid = omp_cpuids.split("|")[self.rank]
|
||||
local_dp_rank = self.parallel_config.data_parallel_rank_local
|
||||
omp_cpuids = omp_cpuids.split("|")
|
||||
if local_dp_rank is not None:
|
||||
world_size = self.parallel_config.world_size
|
||||
omp_cpuids = omp_cpuids[local_dp_rank *
|
||||
world_size:(local_dp_rank + 1) *
|
||||
world_size]
|
||||
self.local_omp_cpuid = omp_cpuids[self.rank]
|
||||
|
||||
if self.local_omp_cpuid != "all":
|
||||
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
|
||||
@@ -162,7 +169,9 @@ class CPUWorker(Worker):
|
||||
# Reserve CPUs for other processes
|
||||
reserve_cpu_num = envs.VLLM_CPU_NUM_OF_RESERVED_CPU
|
||||
if reserve_cpu_num is None:
|
||||
reserve_cpu_num = 1 if self.parallel_config.world_size > 1 else 0
|
||||
need_reserve = (self.parallel_config.world_size > 1 or
|
||||
self.parallel_config.data_parallel_size_local > 1)
|
||||
reserve_cpu_num = 1 if need_reserve else 0
|
||||
assert len(logical_cpu_list) > reserve_cpu_num, (
|
||||
f"VLLM_CPU_NUM_OF_RESERVED_CPU ({reserve_cpu_num}) "
|
||||
f"should less than {len(logical_cpu_list)}.")
|
||||
|
||||
Reference in New Issue
Block a user