[Core] Explicitly manage torch CPU threads in workers (#49919)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-04 10:06:38 -07:00
committed by GitHub
co-authored by Claude
parent 52c0e3cb08
commit 7635a9002b
4 changed files with 207 additions and 25 deletions
+36 -24
View File
@@ -58,6 +58,11 @@ from vllm.utils.system_utils import (
get_mp_context,
set_process_title,
)
from vllm.utils.torch_utils import (
OMP_NUM_THREADS_SET_BY_VLLM,
set_torch_threads_for_runtime,
startup_omp_num_threads,
)
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.executor.abstract import Executor, FailureCallback
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
@@ -122,7 +127,7 @@ class MultiprocExecutor(Executor):
f"_parallel_size ({pcp_size}). "
)
set_multiprocessing_worker_envs()
set_multiprocessing_worker_envs(self.local_world_size)
# use the loopback address get_loopback_ip() for communication.
distributed_init_method = get_distributed_init_method(
@@ -200,6 +205,12 @@ class MultiprocExecutor(Executor):
# Wait for all local workers to be ready.
self.workers = WorkerProc.wait_for_ready(unready_workers)
# The workers have inherited their thread count (see
# set_multiprocessing_worker_envs); this process only schedules, so
# it gets no benefit from torch intra-op parallelism, just CPU
# contention with them.
set_torch_threads_for_runtime()
# Start background thread to monitor worker health if not in headless mode.
if self.monitor_workers:
self.start_worker_monitor()
@@ -1058,32 +1069,33 @@ class WorkerProc:
decorate_logs(process_name)
def set_multiprocessing_worker_envs():
def set_multiprocessing_worker_envs(local_world_size: int = 1):
"""Set up environment variables that should be used when there are workers
in a multiprocessing environment. This should be called by the parent
process before worker processes are created"""
_maybe_force_spawn()
if not current_platform.is_cpu():
# Configure thread parallelism if OMP_NUM_THREADS isn't set
#
# Helps to avoid CPU contention. The default of spawning a thread per
# core combined with multiprocessing for each GPU can have a negative
# impact on performance. The contention is amplified when running in a
# container where CPU limits can cause throttling.
default_omp_num_threads = 1
if (
"OMP_NUM_THREADS" not in os.environ
and (current_parallelism := torch.get_num_threads())
> default_omp_num_threads
):
logger.warning_once(
"Reducing Torch parallelism from %d threads to %d to avoid "
"unnecessary CPU contention. Set OMP_NUM_THREADS in the "
"external environment to tune this value as needed.",
current_parallelism,
default_omp_num_threads,
)
os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
torch.set_num_threads(default_omp_num_threads)
if current_platform.is_cpu() or "OMP_NUM_THREADS" in os.environ:
return
# Choose the workers' thread count here, before they start, since a worker
# must not set its own: `torch.set_num_threads()` spawns the thread pool
# eagerly, and doing that part way through a worker's startup either races
# the dlopen of shared objects or, in a forked worker, deadlocks (libgomp
# is not fork-safe).
num_threads = startup_omp_num_threads(local_world_size)
os.environ["OMP_NUM_THREADS"] = str(num_threads)
os.environ[OMP_NUM_THREADS_SET_BY_VLLM] = "1"
# A spawned worker picks the count up from the environment when it imports
# torch. A forked worker instead inherits it from this process, so set it
# here too. This is safe as long as we don't *use* the pool before forking:
# a forked child whose parent had run a parallel region deadlocks, whereas
# one whose parent merely sized the pool does not.
torch.set_num_threads(num_threads)
logger.debug(
"Set OMP_NUM_THREADS=%d for %d worker process(es).",
num_threads,
local_world_size,
)