[Core] Explicitly manage torch CPU threads in workers (#49919)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -58,6 +58,11 @@ from vllm.utils.system_utils import (
|
||||
get_mp_context,
|
||||
set_process_title,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
OMP_NUM_THREADS_SET_BY_VLLM,
|
||||
set_torch_threads_for_runtime,
|
||||
startup_omp_num_threads,
|
||||
)
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.executor.abstract import Executor, FailureCallback
|
||||
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
|
||||
@@ -122,7 +127,7 @@ class MultiprocExecutor(Executor):
|
||||
f"_parallel_size ({pcp_size}). "
|
||||
)
|
||||
|
||||
set_multiprocessing_worker_envs()
|
||||
set_multiprocessing_worker_envs(self.local_world_size)
|
||||
|
||||
# use the loopback address get_loopback_ip() for communication.
|
||||
distributed_init_method = get_distributed_init_method(
|
||||
@@ -200,6 +205,12 @@ class MultiprocExecutor(Executor):
|
||||
# Wait for all local workers to be ready.
|
||||
self.workers = WorkerProc.wait_for_ready(unready_workers)
|
||||
|
||||
# The workers have inherited their thread count (see
|
||||
# set_multiprocessing_worker_envs); this process only schedules, so
|
||||
# it gets no benefit from torch intra-op parallelism, just CPU
|
||||
# contention with them.
|
||||
set_torch_threads_for_runtime()
|
||||
|
||||
# Start background thread to monitor worker health if not in headless mode.
|
||||
if self.monitor_workers:
|
||||
self.start_worker_monitor()
|
||||
@@ -1058,32 +1069,33 @@ class WorkerProc:
|
||||
decorate_logs(process_name)
|
||||
|
||||
|
||||
def set_multiprocessing_worker_envs():
|
||||
def set_multiprocessing_worker_envs(local_world_size: int = 1):
|
||||
"""Set up environment variables that should be used when there are workers
|
||||
in a multiprocessing environment. This should be called by the parent
|
||||
process before worker processes are created"""
|
||||
|
||||
_maybe_force_spawn()
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
# Configure thread parallelism if OMP_NUM_THREADS isn't set
|
||||
#
|
||||
# Helps to avoid CPU contention. The default of spawning a thread per
|
||||
# core combined with multiprocessing for each GPU can have a negative
|
||||
# impact on performance. The contention is amplified when running in a
|
||||
# container where CPU limits can cause throttling.
|
||||
default_omp_num_threads = 1
|
||||
if (
|
||||
"OMP_NUM_THREADS" not in os.environ
|
||||
and (current_parallelism := torch.get_num_threads())
|
||||
> default_omp_num_threads
|
||||
):
|
||||
logger.warning_once(
|
||||
"Reducing Torch parallelism from %d threads to %d to avoid "
|
||||
"unnecessary CPU contention. Set OMP_NUM_THREADS in the "
|
||||
"external environment to tune this value as needed.",
|
||||
current_parallelism,
|
||||
default_omp_num_threads,
|
||||
)
|
||||
os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
|
||||
torch.set_num_threads(default_omp_num_threads)
|
||||
if current_platform.is_cpu() or "OMP_NUM_THREADS" in os.environ:
|
||||
return
|
||||
|
||||
# Choose the workers' thread count here, before they start, since a worker
|
||||
# must not set its own: `torch.set_num_threads()` spawns the thread pool
|
||||
# eagerly, and doing that part way through a worker's startup either races
|
||||
# the dlopen of shared objects or, in a forked worker, deadlocks (libgomp
|
||||
# is not fork-safe).
|
||||
num_threads = startup_omp_num_threads(local_world_size)
|
||||
os.environ["OMP_NUM_THREADS"] = str(num_threads)
|
||||
os.environ[OMP_NUM_THREADS_SET_BY_VLLM] = "1"
|
||||
|
||||
# A spawned worker picks the count up from the environment when it imports
|
||||
# torch. A forked worker instead inherits it from this process, so set it
|
||||
# here too. This is safe as long as we don't *use* the pool before forking:
|
||||
# a forked child whose parent had run a parallel region deadlocks, whereas
|
||||
# one whose parent merely sized the pool does not.
|
||||
torch.set_num_threads(num_threads)
|
||||
logger.debug(
|
||||
"Set OMP_NUM_THREADS=%d for %d worker process(es).",
|
||||
num_threads,
|
||||
local_world_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user