[Core] V1: Use multiprocessing by default (#11074)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
@@ -232,6 +232,10 @@ class LLM:
|
||||
|
||||
self.request_counter = Counter()
|
||||
|
||||
def __del__(self):
|
||||
if self.llm_engine and hasattr(self.llm_engine, "shutdown"):
|
||||
self.llm_engine.shutdown()
|
||||
|
||||
@staticmethod
|
||||
def get_engine_class() -> Type[LLMEngine]:
|
||||
if envs.VLLM_USE_V1:
|
||||
|
||||
+2
-2
@@ -69,7 +69,7 @@ if TYPE_CHECKING:
|
||||
VLLM_SKIP_P2P_CHECK: bool = False
|
||||
VLLM_DISABLED_KERNELS: List[str] = []
|
||||
VLLM_USE_V1: bool = False
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
|
||||
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
|
||||
|
||||
|
||||
@@ -460,7 +460,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
|
||||
# If set, enable multiprocessing in LLM for the V1 code path.
|
||||
"VLLM_ENABLE_V1_MULTIPROCESSING":
|
||||
lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0"))),
|
||||
lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))),
|
||||
"VLLM_LOG_BATCHSIZE_INTERVAL":
|
||||
lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
|
||||
}
|
||||
|
||||
@@ -274,7 +274,20 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
|
||||
file.write = write_with_prefix # type: ignore[method-assign]
|
||||
|
||||
|
||||
def _check_multiproc_method():
|
||||
if (cuda_is_initialized()
|
||||
and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
|
||||
logger.warning("CUDA was previously initialized. We must use "
|
||||
"the `spawn` multiprocessing start method. Setting "
|
||||
"VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
|
||||
"See https://docs.vllm.ai/en/latest/getting_started/"
|
||||
"debugging.html#python-multiprocessing "
|
||||
"for more information.")
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
def get_mp_context():
|
||||
_check_multiproc_method()
|
||||
mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
|
||||
return multiprocessing.get_context(mp_method)
|
||||
|
||||
@@ -284,12 +297,7 @@ def set_multiprocessing_worker_envs(parallel_config):
|
||||
in a multiprocessing environment. This should be called by the parent
|
||||
process before worker processes are created"""
|
||||
|
||||
if (cuda_is_initialized()
|
||||
and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"):
|
||||
logger.warning("CUDA was previously initialized. We must use "
|
||||
"the `spawn` multiprocessing start method. Setting "
|
||||
"VLLM_WORKER_MULTIPROC_METHOD to 'spawn'.")
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
_check_multiproc_method()
|
||||
|
||||
# Configure thread parallelism if OMP_NUM_THREADS isn't set
|
||||
#
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import multiprocessing
|
||||
import pickle
|
||||
import queue
|
||||
import signal
|
||||
@@ -13,6 +12,7 @@ import zmq.asyncio
|
||||
from msgspec import msgpack
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.executor.multiproc_worker_utils import get_mp_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.v1.core.scheduler import Scheduler
|
||||
@@ -210,11 +210,7 @@ class EngineCoreProc(EngineCore):
|
||||
output_path: str,
|
||||
ready_path: str,
|
||||
) -> EngineCoreProcHandle:
|
||||
# The current process might have CUDA context,
|
||||
# so we need to spawn a new process.
|
||||
# NOTE(rob): this is a problem for using EngineCoreProc w/
|
||||
# LLM, since we need a if __name__ == "__main__" guard.
|
||||
context = multiprocessing.get_context("spawn")
|
||||
context = get_mp_context()
|
||||
|
||||
process_kwargs = {
|
||||
"input_path": input_path,
|
||||
|
||||
@@ -159,10 +159,16 @@ class MPClient(EngineCoreClient):
|
||||
atexit.register(self.shutdown)
|
||||
|
||||
def shutdown(self):
|
||||
# During final garbage collection in process shutdown, atexit may be
|
||||
# None.
|
||||
if atexit:
|
||||
# in case shutdown gets called via __del__ first
|
||||
atexit.unregister(self.shutdown)
|
||||
|
||||
# Shut down the zmq context.
|
||||
self.ctx.destroy(linger=0)
|
||||
|
||||
if hasattr(self, "proc_handle"):
|
||||
if hasattr(self, "proc_handle") and self.proc_handle:
|
||||
# Shutdown the process if needed.
|
||||
if self.proc_handle.proc.is_alive():
|
||||
self.proc_handle.proc.terminate()
|
||||
@@ -178,8 +184,9 @@ class MPClient(EngineCoreClient):
|
||||
]
|
||||
for ipc_socket in ipc_sockets:
|
||||
socket_file = ipc_socket.replace("ipc://", "")
|
||||
if os.path.exists(socket_file):
|
||||
if os and os.path.exists(socket_file):
|
||||
os.remove(socket_file)
|
||||
self.proc_handle = None
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
@@ -196,3 +196,10 @@ class LLMEngine:
|
||||
f"found type: {type(tokenizer_group)}")
|
||||
|
||||
return tokenizer_group
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
def shutdown(self):
|
||||
if engine_core := getattr(self, "engine_core", None):
|
||||
engine_core.shutdown()
|
||||
|
||||
@@ -163,6 +163,10 @@ class MultiprocExecutor:
|
||||
termination and kill signals if needed."""
|
||||
|
||||
def wait_for_termination(procs, timeout):
|
||||
if not time:
|
||||
# If we are in late stage shutdown, the interpreter may replace
|
||||
# `time` with `None`.
|
||||
return all(not proc.is_alive() for proc in procs)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if all(not proc.is_alive() for proc in procs):
|
||||
@@ -187,10 +191,14 @@ class MultiprocExecutor:
|
||||
for w in self.workers:
|
||||
# Remove the zmq ipc socket file
|
||||
socket_path = w.ready_path.replace("ipc://", "")
|
||||
if os.path.exists(socket_path):
|
||||
if os and os.path.exists(socket_path):
|
||||
os.remove(socket_path)
|
||||
|
||||
def shutdown(self):
|
||||
if atexit:
|
||||
# in case shutdown was called explicitly, we don't need to call it
|
||||
# again
|
||||
atexit.unregister(self.shutdown)
|
||||
"""Properly shut down the executor and its workers"""
|
||||
if (hasattr(self, 'workers') and self.workers is not None):
|
||||
for w in self.workers: #TODO: not sure if needed
|
||||
|
||||
Reference in New Issue
Block a user