[Core] Explicitly manage torch CPU threads in workers (#49919)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-04 10:06:38 -07:00
committed by GitHub
co-authored by Claude
parent 52c0e3cb08
commit 7635a9002b
4 changed files with 207 additions and 25 deletions
+116
View File
@@ -4,6 +4,7 @@ import contextlib
import importlib.metadata
import os
import random
import sys
import threading
from collections.abc import Callable, Collection
from typing import TYPE_CHECKING, Any, TypeVar
@@ -150,6 +151,121 @@ def set_default_torch_dtype(dtype: torch.dtype):
torch.set_default_dtype(old_dtype)
def _cgroup_cpu_limit() -> float | None:
"""Effective CPU quota of this process's cgroup, None if unlimited.
Resolves the process's own cgroup from /proc/self/cgroup and takes the
tightest cpu.max (v2) or cfs quota (v1) along the hierarchy.
"""
limit: float | None = None
try:
with open("/proc/self/cgroup") as f:
entries = [line.strip().split(":", 2) for line in f]
def visit(base: str, rel_path: str, read_quota) -> None:
nonlocal limit
path = rel_path
while path:
quota = read_quota(os.path.join(base, path.lstrip("/")))
if quota is not None:
limit = quota if limit is None else min(limit, quota)
path = path.rsplit("/", 1)[0]
def read_v2(cg_dir: str) -> float | None:
try:
with open(os.path.join(cg_dir, "cpu.max")) as f:
quota, period = f.read().split()
return None if quota == "max" else float(quota) / float(period)
except (OSError, ValueError):
return None
def read_v1(cg_dir: str) -> float | None:
try:
with open(os.path.join(cg_dir, "cpu.cfs_quota_us")) as f:
quota = int(f.read())
if quota <= 0:
return None
with open(os.path.join(cg_dir, "cpu.cfs_period_us")) as f:
return quota / int(f.read())
except (OSError, ValueError):
return None
for entry in entries:
if len(entry) != 3:
continue
_, controllers, rel_path = entry
if controllers == "": # cgroup v2
visit("/sys/fs/cgroup", rel_path, read_v2)
elif "cpu" in controllers.split(","): # cgroup v1
visit("/sys/fs/cgroup/cpu", rel_path, read_v1)
except OSError:
pass
return limit
def available_cpu_count() -> int:
"""CPUs actually usable by this process: scheduling affinity capped by
the cgroup CPU quota (unlike `os.cpu_count()`, which is quota-blind)."""
if sys.platform != "linux":
return os.cpu_count() or 1
count = len(os.sched_getaffinity(0))
limit = _cgroup_cpu_limit()
if limit is not None:
count = min(count, int(limit))
return max(1, count)
# Marks OMP_NUM_THREADS as chosen by vLLM for its worker processes rather than
# set by the user, so a worker knows it may drop the value once startup is done.
OMP_NUM_THREADS_SET_BY_VLLM = "VLLM_OMP_NUM_THREADS_SET_BY_VLLM"
def startup_omp_num_threads(num_local_procs: int) -> int:
"""Thread count for a worker process's startup work (weight loading).
Weight loading does CPU-parallel work, so workers benefit from more than
one thread, but only a bounded share of the CPUs this node's workers may
actually use: torch's default is the host core count, which ignores both
scheduling affinity and any cgroup CPU quota, and doesn't account for the
other workers sharing the node.
"""
return max(1, available_cpu_count() // max(1, num_local_procs))
def set_torch_threads_for_runtime() -> None:
"""Set torch intra-op threads to 1 for steady-state serving.
Any multi-threaded torch CPU op in the engine hot loop leaves the OMP
workers spin-waiting after each parallel region, stealing cycles from the
step's serial code (and burning cgroup CPU quota in containers). No
steady-state CPU op benefits from intra-op parallelism.
Respects an externally-set OMP_NUM_THREADS.
"""
if (
omp_num_threads := os.environ.get("OMP_NUM_THREADS")
) is not None and os.environ.get(OMP_NUM_THREADS_SET_BY_VLLM) != "1":
try:
if int(omp_num_threads) > 1:
logger.warning_once(
"OMP_NUM_THREADS=%s is set; leaving Torch threads at %d "
"for serving. Multi-threaded torch CPU ops during serving "
"can degrade performance through spin-wait contention and "
"cgroup CPU-quota throttling.",
omp_num_threads,
torch.get_num_threads(),
)
except ValueError:
pass
return
if torch.get_num_threads() != 1:
logger.info_once(
"Reducing Torch threads from %d to 1 for serving. Set "
"OMP_NUM_THREADS in the external environment to override.",
torch.get_num_threads(),
)
torch.set_num_threads(1)
@contextlib.contextmanager
def set_default_torch_num_threads(num_threads: int | None = None):
"""