[Core] Explicitly manage torch CPU threads in workers (#49919)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import contextlib
|
||||
import importlib.metadata
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Callable, Collection
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
@@ -150,6 +151,121 @@ def set_default_torch_dtype(dtype: torch.dtype):
|
||||
torch.set_default_dtype(old_dtype)
|
||||
|
||||
|
||||
def _cgroup_cpu_limit() -> float | None:
|
||||
"""Effective CPU quota of this process's cgroup, None if unlimited.
|
||||
|
||||
Resolves the process's own cgroup from /proc/self/cgroup and takes the
|
||||
tightest cpu.max (v2) or cfs quota (v1) along the hierarchy.
|
||||
"""
|
||||
limit: float | None = None
|
||||
try:
|
||||
with open("/proc/self/cgroup") as f:
|
||||
entries = [line.strip().split(":", 2) for line in f]
|
||||
|
||||
def visit(base: str, rel_path: str, read_quota) -> None:
|
||||
nonlocal limit
|
||||
path = rel_path
|
||||
while path:
|
||||
quota = read_quota(os.path.join(base, path.lstrip("/")))
|
||||
if quota is not None:
|
||||
limit = quota if limit is None else min(limit, quota)
|
||||
path = path.rsplit("/", 1)[0]
|
||||
|
||||
def read_v2(cg_dir: str) -> float | None:
|
||||
try:
|
||||
with open(os.path.join(cg_dir, "cpu.max")) as f:
|
||||
quota, period = f.read().split()
|
||||
return None if quota == "max" else float(quota) / float(period)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def read_v1(cg_dir: str) -> float | None:
|
||||
try:
|
||||
with open(os.path.join(cg_dir, "cpu.cfs_quota_us")) as f:
|
||||
quota = int(f.read())
|
||||
if quota <= 0:
|
||||
return None
|
||||
with open(os.path.join(cg_dir, "cpu.cfs_period_us")) as f:
|
||||
return quota / int(f.read())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
for entry in entries:
|
||||
if len(entry) != 3:
|
||||
continue
|
||||
_, controllers, rel_path = entry
|
||||
if controllers == "": # cgroup v2
|
||||
visit("/sys/fs/cgroup", rel_path, read_v2)
|
||||
elif "cpu" in controllers.split(","): # cgroup v1
|
||||
visit("/sys/fs/cgroup/cpu", rel_path, read_v1)
|
||||
except OSError:
|
||||
pass
|
||||
return limit
|
||||
|
||||
|
||||
def available_cpu_count() -> int:
|
||||
"""CPUs actually usable by this process: scheduling affinity capped by
|
||||
the cgroup CPU quota (unlike `os.cpu_count()`, which is quota-blind)."""
|
||||
if sys.platform != "linux":
|
||||
return os.cpu_count() or 1
|
||||
count = len(os.sched_getaffinity(0))
|
||||
limit = _cgroup_cpu_limit()
|
||||
if limit is not None:
|
||||
count = min(count, int(limit))
|
||||
return max(1, count)
|
||||
|
||||
|
||||
# Marks OMP_NUM_THREADS as chosen by vLLM for its worker processes rather than
|
||||
# set by the user, so a worker knows it may drop the value once startup is done.
|
||||
OMP_NUM_THREADS_SET_BY_VLLM = "VLLM_OMP_NUM_THREADS_SET_BY_VLLM"
|
||||
|
||||
|
||||
def startup_omp_num_threads(num_local_procs: int) -> int:
|
||||
"""Thread count for a worker process's startup work (weight loading).
|
||||
|
||||
Weight loading does CPU-parallel work, so workers benefit from more than
|
||||
one thread, but only a bounded share of the CPUs this node's workers may
|
||||
actually use: torch's default is the host core count, which ignores both
|
||||
scheduling affinity and any cgroup CPU quota, and doesn't account for the
|
||||
other workers sharing the node.
|
||||
"""
|
||||
return max(1, available_cpu_count() // max(1, num_local_procs))
|
||||
|
||||
|
||||
def set_torch_threads_for_runtime() -> None:
|
||||
"""Set torch intra-op threads to 1 for steady-state serving.
|
||||
|
||||
Any multi-threaded torch CPU op in the engine hot loop leaves the OMP
|
||||
workers spin-waiting after each parallel region, stealing cycles from the
|
||||
step's serial code (and burning cgroup CPU quota in containers). No
|
||||
steady-state CPU op benefits from intra-op parallelism.
|
||||
Respects an externally-set OMP_NUM_THREADS.
|
||||
"""
|
||||
if (
|
||||
omp_num_threads := os.environ.get("OMP_NUM_THREADS")
|
||||
) is not None and os.environ.get(OMP_NUM_THREADS_SET_BY_VLLM) != "1":
|
||||
try:
|
||||
if int(omp_num_threads) > 1:
|
||||
logger.warning_once(
|
||||
"OMP_NUM_THREADS=%s is set; leaving Torch threads at %d "
|
||||
"for serving. Multi-threaded torch CPU ops during serving "
|
||||
"can degrade performance through spin-wait contention and "
|
||||
"cgroup CPU-quota throttling.",
|
||||
omp_num_threads,
|
||||
torch.get_num_threads(),
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
return
|
||||
if torch.get_num_threads() != 1:
|
||||
logger.info_once(
|
||||
"Reducing Torch threads from %d to 1 for serving. Set "
|
||||
"OMP_NUM_THREADS in the external environment to override.",
|
||||
torch.get_num_threads(),
|
||||
)
|
||||
torch.set_num_threads(1)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_default_torch_num_threads(num_threads: int | None = None):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user