Revert "[Startup] Parallelize torch/transformers import + weight prefetch + forkserver prewarm" (#40438)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -5,80 +5,10 @@
|
||||
Note that all future modules must be lazily loaded within main
|
||||
to avoid certain eager import breakage."""
|
||||
|
||||
import contextlib
|
||||
import importlib.metadata
|
||||
import os
|
||||
import sys
|
||||
import threading as _threading
|
||||
|
||||
|
||||
# [startup] Kick off torch + transformers .so/module loading in a background
|
||||
# thread before we touch vllm.logger (which pulls vllm/__init__.py ->
|
||||
# vllm.env_override -> `import torch` on the main thread). Python import
|
||||
# lock serializes the same-module import across threads, but the .so dlopen
|
||||
# inside torch's init releases the GIL during file I/O. Main thread's
|
||||
# non-torch imports (vllm.envs submodules, stdlib, fastapi, etc.) can make
|
||||
# progress on the CPU while the background thread pays the ~2 s of cuda
|
||||
# .so loading. `import transformers` is also ~2 s of cold-disk work and
|
||||
# depends on torch; chain it after torch in the same thread so subsequent
|
||||
# `from transformers import ...` lines on the main thread hit a warm
|
||||
# module cache.
|
||||
def _bg_preload_torch() -> None:
|
||||
try:
|
||||
import torch # noqa: F401
|
||||
except Exception:
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
import transformers # noqa: F401
|
||||
|
||||
|
||||
_threading.Thread(
|
||||
target=_bg_preload_torch, daemon=True, name="vllm-torch-preload"
|
||||
).start()
|
||||
|
||||
|
||||
# [startup] Pre-spawn EngineCore via forkserver preload, in a background
|
||||
# thread. Only fires for `vllm serve` (the only subcommand that spawns a
|
||||
# long-running EngineCore). The forkserver process is forked once and
|
||||
# preloaded with vllm.v1.engine.async_llm (~3-5 s of imports). When
|
||||
# AsyncLLM.from_vllm_config later runs, Process.start() forks from the
|
||||
# already-warm forkserver instead of paying spawn() cost (~5 s in child
|
||||
# for fresh Python + imports).
|
||||
#
|
||||
# Kicking the preload in a BG thread lets the ~3-5 s ensure_running cost
|
||||
# overlap with APIServer's argparse + config resolution (~5-10 s on cold
|
||||
# disk). Default cli_env_setup sets spawn; we override to forkserver
|
||||
# before that runs so the path is consistent.
|
||||
def _bg_prewarm_forkserver() -> None:
|
||||
try:
|
||||
import multiprocessing
|
||||
import multiprocessing.forkserver as forkserver
|
||||
|
||||
# set_start_method MUST be called before ensure_running. It also
|
||||
# can only be called once per process; any later override by
|
||||
# vllm's build_async_engine_client will just see the existing
|
||||
# setting.
|
||||
multiprocessing.set_start_method("forkserver", force=False)
|
||||
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
|
||||
forkserver.ensure_running()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "serve":
|
||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")
|
||||
# daemon=True so early CLI exits (bad args, --help, import errors)
|
||||
# don't hang waiting for ensure_running(). The forkserver subprocess
|
||||
# itself is tracked by module-level state in multiprocessing.forkserver
|
||||
# and survives this thread exiting; subsequent spawn() calls reuse it.
|
||||
_threading.Thread(
|
||||
target=_bg_prewarm_forkserver,
|
||||
daemon=True,
|
||||
name="vllm-forkserver-prewarm",
|
||||
).start()
|
||||
|
||||
|
||||
from vllm.logger import init_logger # noqa: E402
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user