[CPU] Refactor CPU affinity and memory management (#39781)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -0,0 +1,173 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
|
||||
import psutil
|
||||
import regex as re
|
||||
|
||||
DEVICE_CONTROL_ENV_VAR = "CPU_VISIBLE_MEMORY_NODES"
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogicalCPUInfo:
|
||||
id: int = -1
|
||||
physical_core: int = -1
|
||||
numa_node: int = -1
|
||||
|
||||
@classmethod
|
||||
def _int(cls, value: str) -> int:
|
||||
try:
|
||||
int_value = int(value)
|
||||
except Exception:
|
||||
int_value = -1
|
||||
return int_value
|
||||
|
||||
@staticmethod
|
||||
def json_decoder(obj_dict: dict):
|
||||
id = obj_dict.get("cpu")
|
||||
physical_core = obj_dict.get("core")
|
||||
numa_node = obj_dict.get("node")
|
||||
|
||||
if not (id is None or physical_core is None or numa_node is None):
|
||||
return LogicalCPUInfo(
|
||||
id=LogicalCPUInfo._int(id),
|
||||
physical_core=LogicalCPUInfo._int(physical_core),
|
||||
numa_node=LogicalCPUInfo._int(numa_node),
|
||||
)
|
||||
else:
|
||||
return obj_dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryNodeInfo:
|
||||
total_memory: int = -1
|
||||
available_memory: int = -1
|
||||
|
||||
|
||||
def get_memory_affinity(pid: int = 0) -> list[int]:
|
||||
pid = os.getpid() if pid == 0 else pid
|
||||
path = f"/proc/{pid}/status"
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if line.startswith("Mems_allowed_list:"):
|
||||
# Extract the string part (e.g., "0-1,3")
|
||||
raw_list = line.split(":")[1].strip()
|
||||
return parse_id_list(raw_list)
|
||||
return []
|
||||
|
||||
|
||||
def parse_id_list(raw_str: str) -> list[int]:
|
||||
"""Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8]"""
|
||||
result: list[int] = []
|
||||
if not raw_str:
|
||||
return result
|
||||
|
||||
for part in raw_str.split(","):
|
||||
if "-" in part:
|
||||
start, end = map(int, part.split("-"))
|
||||
result.extend(range(start, end + 1))
|
||||
else:
|
||||
result.append(int(part))
|
||||
return sorted(list(set(result)))
|
||||
|
||||
|
||||
def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo:
|
||||
if platform.system() == "Darwin":
|
||||
# MacOS has no memory node
|
||||
return MemoryNodeInfo(
|
||||
total_memory=psutil.virtual_memory().total,
|
||||
available_memory=psutil.virtual_memory().available,
|
||||
)
|
||||
|
||||
meminfo_path = f"/sys/devices/system/node/node{node_id}/meminfo"
|
||||
if not os.path.exists(meminfo_path):
|
||||
raise RuntimeError(f"{meminfo_path} doesn't exit.")
|
||||
|
||||
meminfo = {}
|
||||
with open(meminfo_path) as f:
|
||||
for line in f:
|
||||
# Each line looks like: "Node 0 MemTotal: 97421888 kB"
|
||||
parts = line.split()
|
||||
key = parts[2].rstrip(":")
|
||||
# convert to Bytes
|
||||
value = int(parts[3]) * 1024
|
||||
meminfo[key] = value
|
||||
|
||||
total_memory = meminfo["MemTotal"]
|
||||
free_memory = meminfo["MemFree"]
|
||||
active_file_memory = meminfo["Active(file)"]
|
||||
inactive_file_memory = meminfo["Inactive(file)"]
|
||||
reclaimable_memory = meminfo["SReclaimable"]
|
||||
available_memory = (
|
||||
free_memory + active_file_memory + inactive_file_memory + reclaimable_memory
|
||||
)
|
||||
|
||||
return MemoryNodeInfo(
|
||||
total_memory=total_memory,
|
||||
available_memory=available_memory,
|
||||
)
|
||||
|
||||
|
||||
def get_allowed_cpu_list() -> list[LogicalCPUInfo]:
|
||||
cpu_list = _get_cpu_list()
|
||||
if platform.system() == "Darwin":
|
||||
return cpu_list
|
||||
|
||||
global_allowed_cpu_id_list = os.sched_getaffinity(0)
|
||||
logical_cpu_list = [x for x in cpu_list if x.id in global_allowed_cpu_id_list]
|
||||
|
||||
return logical_cpu_list
|
||||
|
||||
|
||||
def get_visible_memory_node() -> list[int]:
|
||||
if platform.system() == "Darwin":
|
||||
return [0]
|
||||
|
||||
allowed_memory_node_list = get_memory_affinity()
|
||||
|
||||
env_key = DEVICE_CONTROL_ENV_VAR
|
||||
if (
|
||||
("VLLM_CPU_SIM_MULTI_NUMA" not in os.environ)
|
||||
and env_key in os.environ
|
||||
and os.environ[env_key] != ""
|
||||
):
|
||||
visible_nodes = [int(s) for s in os.environ[env_key].split(",")]
|
||||
visible_nodes = [
|
||||
node for node in visible_nodes if node in allowed_memory_node_list
|
||||
]
|
||||
return visible_nodes
|
||||
|
||||
return allowed_memory_node_list
|
||||
|
||||
|
||||
@cache
|
||||
def _get_cpu_list() -> list[LogicalCPUInfo]:
|
||||
if platform.system() == "Darwin":
|
||||
# For MacOS, no user-level CPU affinity and SMT, return all CPUs
|
||||
cpu_count = os.cpu_count()
|
||||
assert cpu_count
|
||||
return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)]
|
||||
|
||||
lscpu_output = subprocess.check_output(
|
||||
"lscpu -J -e=CPU,CORE,NODE", shell=True, text=True
|
||||
)
|
||||
|
||||
# For platform without NUMA, replace '-' to '0'
|
||||
lscpu_output = re.sub(r'"node":\s*-\s*(,|\n)', r'"node": 0\1', lscpu_output)
|
||||
|
||||
logical_cpu_list: list[LogicalCPUInfo] = json.loads(
|
||||
lscpu_output, object_hook=LogicalCPUInfo.json_decoder
|
||||
)["cpus"]
|
||||
|
||||
# Filter CPUs with invalid attributes
|
||||
logical_cpu_list = [
|
||||
x for x in logical_cpu_list if -1 not in (x.id, x.physical_core, x.numa_node)
|
||||
]
|
||||
|
||||
return logical_cpu_list
|
||||
+269
-185
@@ -5,196 +5,280 @@ Copyright (c) 2026 Red Hat Inc
|
||||
Copyright (c) 2026 Cambridge Greys Ltd
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import vllm.utils.cpu_resource_utils as cr_utils
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.cpu_resource_utils import LogicalCPUInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _int(arg):
|
||||
"""Relaxed parsing of ints which handles a - instead of a number.
|
||||
The lscpu json may contain that for nodes in some cases. If that
|
||||
is the case we parse it to zero
|
||||
"""
|
||||
try:
|
||||
if int(arg) >= 0:
|
||||
return int(arg)
|
||||
except ValueError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def parse_mask(mask):
|
||||
"""Expand a X-Y,Z list"""
|
||||
result = []
|
||||
for token in mask.split(","):
|
||||
try:
|
||||
start, finish = token.split("-")
|
||||
if int(start) > int(finish):
|
||||
raise IndexError("Invalid Indexes for cpu ranges")
|
||||
for cpu in range(int(start), int(finish) + 1):
|
||||
result.append(cpu)
|
||||
except ValueError:
|
||||
result.append(int(token))
|
||||
return set(result)
|
||||
|
||||
|
||||
def _get_default_affinity() -> set[int]:
|
||||
"""Get the set of CPUs the process is allowed to run on."""
|
||||
if hasattr(os, "sched_getaffinity"):
|
||||
return os.sched_getaffinity(0)
|
||||
# macOS does not support sched_getaffinity; fall back to cpu_count
|
||||
cpu_count = os.cpu_count() or 1
|
||||
return set(range(cpu_count))
|
||||
|
||||
|
||||
def _get_cpu_topology_json() -> bytes:
|
||||
"""Get CPU topology as JSON.
|
||||
|
||||
On Linux this uses ``lscpu -Je``. On other platforms (e.g. macOS) we
|
||||
synthesize a simple topology where every logical CPU is its own core
|
||||
on NUMA node 0, which is sufficient for the OMP place-list builder.
|
||||
"""
|
||||
if platform.system() == "Linux":
|
||||
return subprocess.run(["lscpu", "-Je"], check=True, capture_output=True).stdout
|
||||
|
||||
# Fallback for non-Linux (macOS, etc.)
|
||||
cpu_count = os.cpu_count() or 1
|
||||
cpus = []
|
||||
for i in range(cpu_count):
|
||||
cpus.append({"cpu": str(i), "core": str(i), "node": "0"})
|
||||
return json.dumps({"cpus": cpus}).encode()
|
||||
|
||||
|
||||
def enumerate_resources(resource_map, mask=None, allowed=None):
|
||||
"""Enumerate system resources"""
|
||||
if allowed is None:
|
||||
allowed = _get_default_affinity()
|
||||
if mask is not None:
|
||||
allowed = allowed & mask
|
||||
|
||||
try:
|
||||
allowed_nodes = parse_mask(os.environ["CPU_VISIBLE_MEMORY_NODES"])
|
||||
except KeyError:
|
||||
allowed_nodes = None
|
||||
|
||||
lscpu: dict[str, dict] = {"cpus": {}, "cores": {}, "nodes": {}}
|
||||
for cpu in resource_map["cpus"]:
|
||||
cpunum = int(cpu["cpu"])
|
||||
if (
|
||||
cpunum in allowed
|
||||
and cpunum >= 0
|
||||
and (allowed_nodes is None or _int(cpu["node"]) in allowed_nodes)
|
||||
):
|
||||
lscpu["cpus"][cpunum] = [cpu]
|
||||
core = _int(cpu["core"])
|
||||
if lscpu["cores"].get(core, None) is None:
|
||||
lscpu["cores"][core] = [cpu]
|
||||
else:
|
||||
lscpu["cores"][core].append(cpu)
|
||||
node = _int(cpu["node"])
|
||||
if lscpu["nodes"].get(node, None) is None:
|
||||
lscpu["nodes"][node] = [cpu]
|
||||
else:
|
||||
lscpu["nodes"][node].append(cpu)
|
||||
return lscpu
|
||||
|
||||
|
||||
def produce_cpu_list(cpus, smt=1):
|
||||
"""Produce a CPU list with/without SMT pairs - main cpu list case"""
|
||||
mask: list[int] = []
|
||||
for key, value in cpus.items():
|
||||
exists = 0
|
||||
for cpu in mask:
|
||||
if cpu == value[0]["core"]:
|
||||
exists += 1
|
||||
break
|
||||
if exists < smt:
|
||||
mask.append(int(key))
|
||||
return {"mask": set(mask), "available": True}
|
||||
|
||||
|
||||
def produce_cpu_sublist(scpus, smt=1):
|
||||
"""Produce a CPU list with/without SMT pairs - resource leaf case"""
|
||||
cpu_list: list[dict] = []
|
||||
for value in scpus:
|
||||
exists = 0
|
||||
for cpu in cpu_list:
|
||||
if int(cpu["core"]) == int(value["core"]):
|
||||
exists += 1
|
||||
break
|
||||
if exists < smt:
|
||||
cpu_list.append(value)
|
||||
mask = []
|
||||
for cpu in cpu_list:
|
||||
mask.append(int(cpu["cpu"]))
|
||||
|
||||
return {"mask": set(mask), "available": True}
|
||||
|
||||
|
||||
def create_omp_places(resources, strategy, smt=True):
|
||||
"""Parse CPU topology and generate possible CPU masks"""
|
||||
omp_places = []
|
||||
if strategy == "all":
|
||||
omp_places.append(produce_cpu_list(resources["cpus"], smt))
|
||||
elif strategy == "cores":
|
||||
for value in resources["cores"].values():
|
||||
omp_places.append(produce_cpu_sublist(value, smt))
|
||||
elif strategy == "nodes":
|
||||
for value in resources["nodes"].values():
|
||||
omp_places.append(produce_cpu_sublist(value, smt))
|
||||
else:
|
||||
raise NotImplementedError("Unknown strategy")
|
||||
|
||||
return omp_places
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class OMPProcessManager:
|
||||
"""OMP aware wrapper to run mp Process()"""
|
||||
def __init__(self, config: "VllmConfig"):
|
||||
if not current_platform.is_cpu():
|
||||
return
|
||||
|
||||
def __init__(self, strategy="nodes", smt=1, mock=None, affinity=None):
|
||||
self.strategy = strategy
|
||||
self.smt = smt
|
||||
self.omp_places = []
|
||||
vllm_mask = os.environ.get("VLLM_CPU_OMP_THREADS_BIND", None)
|
||||
self.setup_omp = vllm_mask != "nobind"
|
||||
if self.setup_omp:
|
||||
omp_places = []
|
||||
if vllm_mask is not None:
|
||||
masks = []
|
||||
for spec in vllm_mask.split("|"):
|
||||
masks.append(parse_mask(spec))
|
||||
self.local_world_size = config.parallel_config.local_world_size
|
||||
self.local_dp_rank = config.parallel_config.data_parallel_rank_local
|
||||
# This is a bit tricky because the internal DP size
|
||||
# is always 1 for non-MoE models
|
||||
self.internal_dp_size = config.parallel_config._api_process_count
|
||||
|
||||
self.simulate_multi_node = os.environ.get("VLLM_CPU_SIM_MULTI_NUMA", "0") != "0"
|
||||
ld_preload_str = os.getenv("LD_PRELOAD", "")
|
||||
self.use_iomp = "libiomp" in ld_preload_str or "libomp" in ld_preload_str
|
||||
self.use_gomp = "libgomp" in ld_preload_str
|
||||
|
||||
assert not (self.use_iomp and self.use_gomp)
|
||||
|
||||
# at least reserve 1/local_world_size(for ARM) core for scheduler
|
||||
# proc as always use MP executor
|
||||
# TODO: make scheduler proc sleep when idle
|
||||
self.reserve_cpu_num = (
|
||||
self.local_world_size
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
else 1
|
||||
)
|
||||
# reserve at one more core for nixl_connector under p/d case
|
||||
if config.kv_transfer_config:
|
||||
self.reserve_cpu_num += 1
|
||||
|
||||
if envs.VLLM_CPU_NUM_OF_RESERVED_CPU is not None:
|
||||
if self.reserve_cpu_num > envs.VLLM_CPU_NUM_OF_RESERVED_CPU:
|
||||
msg = (
|
||||
f"VLLM_CPU_NUM_OF_RESERVED_CPU is less than "
|
||||
"the minimum requirement"
|
||||
f": {self.reserve_cpu_num} cores"
|
||||
)
|
||||
logger.warning(msg=msg)
|
||||
self.reserve_cpu_num = envs.VLLM_CPU_NUM_OF_RESERVED_CPU
|
||||
|
||||
self._parse_omp_threads_bind_env()
|
||||
|
||||
assert not self.simulate_multi_node or self.auto_setup
|
||||
|
||||
@contextmanager
|
||||
def configure_omp_envs(self, rank: int, local_rank: int):
|
||||
if not current_platform.is_cpu() or self.skip_setup:
|
||||
yield
|
||||
return
|
||||
|
||||
envs_dict = {}
|
||||
cpu_list = [str(i) for i in self.cpu_lists[local_rank]]
|
||||
envs_dict["OMP_NUM_THREADS"] = str(len(cpu_list))
|
||||
if self.use_iomp:
|
||||
# set IOMP envs
|
||||
cpu_list_str = ",".join(cpu_list)
|
||||
envs_dict["KMP_AFFINITY"] = (
|
||||
f"granularity=fine,explicit,proclist=[{cpu_list_str}]"
|
||||
)
|
||||
# The time(milliseconds) that a thread should wait after
|
||||
# completing the execution of a parallel region, before sleeping.
|
||||
envs_dict["KMP_BLOCKTIME"] = "1"
|
||||
# Prevents the CPU to run into low performance state
|
||||
envs_dict["KMP_TPAUSE"] = "0"
|
||||
# Provides fine granularity parallelism
|
||||
envs_dict["KMP_FORKJOIN_BARRIER_PATTERN"] = "dist,dist"
|
||||
envs_dict["KMP_PLAIN_BARRIER_PATTERN"] = "dist,dist"
|
||||
envs_dict["KMP_REDUCTION_BARRIER_PATTERN"] = "dist,dist"
|
||||
elif self.use_gomp:
|
||||
# set GOMP envs
|
||||
# likes '0 1 2 ...'
|
||||
cpu_list_str = " ".join(cpu_list)
|
||||
envs_dict["GOMP_CPU_AFFINITY"] = cpu_list_str
|
||||
else:
|
||||
# set OMP envs
|
||||
# likes '{0,1,2,...}'
|
||||
cpu_list_str = ",".join(cpu_list)
|
||||
envs_dict["OMP_PLACES"] = f"{{{cpu_list_str}}}"
|
||||
envs_dict["OMP_PROC_BIND"] = "true"
|
||||
|
||||
# backup envs
|
||||
old_envs_dict = {}
|
||||
for k in envs_dict:
|
||||
old_envs_dict[k] = os.environ.get(k)
|
||||
|
||||
try:
|
||||
# set envs
|
||||
for k, v in envs_dict.items():
|
||||
os.environ[k] = v
|
||||
yield
|
||||
finally:
|
||||
# restore old envs
|
||||
for k, v in old_envs_dict.items(): # type: ignore
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
|
||||
def _parse_omp_threads_bind_env(self):
|
||||
vllm_mask = envs.VLLM_CPU_OMP_THREADS_BIND
|
||||
self.skip_setup = vllm_mask == "nobind"
|
||||
self.auto_setup = vllm_mask == "auto"
|
||||
self.reserved_cpu_list = []
|
||||
self.cpu_lists = []
|
||||
|
||||
if self.auto_setup:
|
||||
# auto generate CPU lists
|
||||
cpu_arch = current_platform.get_cpu_architecture()
|
||||
if cpu_arch == CpuArchEnum.POWERPC:
|
||||
# For POWERPC SMT-8/4/2
|
||||
cpu_list, reserve_list = self._get_autobind_cpu_ids(
|
||||
lambda cpus: [cpu for cpu in cpus if cpu.id % 8 < 4]
|
||||
)
|
||||
elif cpu_arch in (CpuArchEnum.X86, CpuArchEnum.S390X):
|
||||
# For x86/S390X SMT-2, use 1 logical CPU per physical core
|
||||
cpu_list, reserve_list = self._get_autobind_cpu_ids(
|
||||
lambda cpus: cpus[-1:]
|
||||
)
|
||||
elif cpu_arch == CpuArchEnum.ARM:
|
||||
# For AArch64, no SMT, use all logical CPU
|
||||
cpu_list, reserve_list = self._get_autobind_cpu_ids(lambda cpus: cpus)
|
||||
else:
|
||||
masks = [None]
|
||||
if mock is None:
|
||||
data = _get_cpu_topology_json()
|
||||
else:
|
||||
with open(mock, mode="rb") as jf:
|
||||
data = jf.read()
|
||||
lscpu = json.loads(data)
|
||||
for mask in masks:
|
||||
resources = enumerate_resources(lscpu, mask, affinity)
|
||||
omp_places.extend(create_omp_places(resources, strategy, smt))
|
||||
self.omp_places = sorted(
|
||||
omp_places,
|
||||
key=lambda p: "{:04d}-{:04d}".format(len(p["mask"]), max(p["mask"])),
|
||||
reverse=True,
|
||||
cpu_list, reserve_list = [], []
|
||||
raise RuntimeError(f"{cpu_arch} doesn't support auto CPU binding.")
|
||||
|
||||
for item in cpu_list:
|
||||
self.cpu_lists.append([x.id for x in item])
|
||||
self.reserved_cpu_list = [x.id for x in reserve_list]
|
||||
elif not self.skip_setup:
|
||||
# user defined CPU lists
|
||||
omp_cpuids_list = vllm_mask.split("|")
|
||||
if self.local_dp_rank is not None:
|
||||
local_dp_rank = self.local_dp_rank
|
||||
world_size = self.local_world_size
|
||||
# Rank mapping [DP, PP, TP]
|
||||
omp_cpuids_list = omp_cpuids_list[
|
||||
local_dp_rank * world_size : (local_dp_rank + 1) * world_size
|
||||
]
|
||||
|
||||
assert len(omp_cpuids_list) == self.local_world_size, (
|
||||
"Given "
|
||||
f"number of CPU id list {omp_cpuids_list} doesn't match "
|
||||
f"local world size {self.local_world_size}."
|
||||
)
|
||||
|
||||
def run(self, what, *args, **kwargs):
|
||||
"""Run arg with correct OMP environment"""
|
||||
if self.setup_omp:
|
||||
for place in self.omp_places:
|
||||
if place["available"]:
|
||||
reserve = int(os.environ.get("VLLM_CPU_NUM_OF_RESERVED_CPU", 0))
|
||||
place["available"] = False
|
||||
# pylint: disable=consider-using-f-string
|
||||
os.environ["OMP_PLACES"] = "{}".format(place["mask"])
|
||||
os.environ["OMP_NUM_THREADS"] = "{}".format(
|
||||
len(place["mask"]) - reserve
|
||||
)
|
||||
os.environ["OMP_PROC_BIND"] = "TRUE"
|
||||
return what(*args, **kwargs)
|
||||
raise IndexError("Out of OMP places")
|
||||
return what(*args, **kwargs)
|
||||
# parse CPU list strings like "5,2-4" to [5, 2, 3, 4]
|
||||
self.cpu_lists = [cr_utils.parse_id_list(s) for s in omp_cpuids_list]
|
||||
else:
|
||||
# skip
|
||||
self.cpu_lists = []
|
||||
|
||||
msg = "OpenMP thread binding info: \n"
|
||||
for i in range(self.local_world_size):
|
||||
msg += f"\tlocal_rank={i}, core ids={self.cpu_lists[i]}\n"
|
||||
msg += f"\treserved_cpus={self.reserved_cpu_list}"
|
||||
logger.info(msg)
|
||||
|
||||
def _get_autobind_cpu_ids(
|
||||
self, cpu_selector: Callable[[list[LogicalCPUInfo]], list[LogicalCPUInfo]]
|
||||
) -> tuple[list[list[LogicalCPUInfo]], list[LogicalCPUInfo]]:
|
||||
"""
|
||||
Return CPU ids to bind based on NUMA nodes, and CPU ids reserved for
|
||||
other processes.
|
||||
Currently for rank N, only CPU ids on the N-th node in available NUMA
|
||||
node list will be selected.
|
||||
Args:
|
||||
cpu_selector: a callable object to select CPUs from a CPU list
|
||||
of a physical core. The input is a LogicalCPUInfo list contains
|
||||
logical CPUs of a physical CPU, sorted by the LogicalCPUInfo.id.
|
||||
A selected LogicalCPUInfo list should be returned.
|
||||
"""
|
||||
|
||||
# this memory node list has been sliced for DP offset
|
||||
allowed_numa_nodes = cr_utils.get_visible_memory_node()
|
||||
logical_cpu_list = cr_utils.get_allowed_cpu_list()
|
||||
|
||||
local_world_size = self.local_world_size
|
||||
assert (
|
||||
len(allowed_numa_nodes) >= local_world_size or self.simulate_multi_node
|
||||
), (
|
||||
f"Not enough allowed NUMA nodes to bind threads of "
|
||||
f"{local_world_size} local CPUWorkers. "
|
||||
f"Allowed NUMA nodes are {allowed_numa_nodes}. "
|
||||
"Please try to bind threads manually or decrease DP/TP/PP."
|
||||
)
|
||||
|
||||
# Generate OMP CPU list for each rank
|
||||
cpu_lists_of_ranks = []
|
||||
reserved_cpu_list = []
|
||||
total_cpu_num = 0
|
||||
for local_rank in range(self.local_world_size):
|
||||
if not self.simulate_multi_node:
|
||||
selected_numa_node = allowed_numa_nodes[local_rank]
|
||||
selected_logical_cpu_list = [
|
||||
x for x in logical_cpu_list if x.numa_node == selected_numa_node
|
||||
]
|
||||
else:
|
||||
world_size_across_dp = self.local_world_size * self.internal_dp_size
|
||||
assert len(logical_cpu_list) >= world_size_across_dp
|
||||
selected_logical_cpu_list = sorted(
|
||||
logical_cpu_list, key=lambda x: x.numa_node
|
||||
)
|
||||
sim_cpu_num_per_node = (
|
||||
len(selected_logical_cpu_list) // world_size_across_dp
|
||||
)
|
||||
assert self.local_dp_rank is not None
|
||||
start_idx = (
|
||||
local_rank + self.local_world_size * self.local_dp_rank
|
||||
) * sim_cpu_num_per_node
|
||||
selected_logical_cpu_list = selected_logical_cpu_list[
|
||||
start_idx : (start_idx + sim_cpu_num_per_node)
|
||||
]
|
||||
|
||||
# Select logical CPUs on same physical cores via cpu_selector
|
||||
core_to_cpus: dict[int, list[LogicalCPUInfo]] = {}
|
||||
for cpu_info in selected_logical_cpu_list:
|
||||
if cpu_info.physical_core not in core_to_cpus:
|
||||
core_to_cpus[cpu_info.physical_core] = []
|
||||
core_to_cpus[cpu_info.physical_core].append(cpu_info)
|
||||
selected_logical_cpu_list = []
|
||||
for cpu_list in core_to_cpus.values():
|
||||
cpu_list = sorted(cpu_list, key=lambda x: x.id)
|
||||
selected_logical_cpu_list.extend(cpu_selector(cpu_list))
|
||||
|
||||
# sort selected cores based on core id
|
||||
selected_logical_cpu_list = sorted(
|
||||
selected_logical_cpu_list, key=lambda x: x.id
|
||||
)
|
||||
|
||||
cpu_lists_of_ranks.append(selected_logical_cpu_list)
|
||||
total_cpu_num += len(selected_logical_cpu_list)
|
||||
|
||||
# Reserve CPUs for other processes
|
||||
if total_cpu_num <= self.reserve_cpu_num:
|
||||
logger.warning(
|
||||
"Selected CPU core number (%s) "
|
||||
"should be greater than reserved CPU core "
|
||||
"number (%s).",
|
||||
total_cpu_num,
|
||||
self.reserve_cpu_num,
|
||||
)
|
||||
return cpu_lists_of_ranks, []
|
||||
|
||||
reserve_num_per_rank = [
|
||||
self.reserve_cpu_num // self.local_world_size
|
||||
] * self.local_world_size
|
||||
# last rank first
|
||||
for i in range(
|
||||
self.local_world_size - 1,
|
||||
self.local_world_size - 1 - self.reserve_cpu_num % self.local_world_size,
|
||||
-1,
|
||||
):
|
||||
reserve_num_per_rank[i] += 1
|
||||
for i in range(self.local_world_size):
|
||||
num = reserve_num_per_rank[i]
|
||||
if num > 0:
|
||||
reserved_cpu_list.extend(cpu_lists_of_ranks[i][-num:])
|
||||
cpu_lists_of_ranks[i] = cpu_lists_of_ranks[i][:-num]
|
||||
|
||||
return cpu_lists_of_ranks, reserved_cpu_list
|
||||
|
||||
Reference in New Issue
Block a user