[Core] V1: Use multiprocessing by default (#11074)

Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Russell Bryant
2024-12-13 16:27:32 -08:00
committed by GitHub
parent 0d8451c3a4
commit 4863e5fba5
10 changed files with 299 additions and 17 deletions
+2 -6
View File
@@ -1,4 +1,3 @@
import multiprocessing
import pickle
import queue
import signal
@@ -13,6 +12,7 @@ import zmq.asyncio
from msgspec import msgpack
from vllm.config import CacheConfig, VllmConfig
from vllm.executor.multiproc_worker_utils import get_mp_context
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext
from vllm.v1.core.scheduler import Scheduler
@@ -210,11 +210,7 @@ class EngineCoreProc(EngineCore):
output_path: str,
ready_path: str,
) -> EngineCoreProcHandle:
# The current process might have CUDA context,
# so we need to spawn a new process.
# NOTE(rob): this is a problem for using EngineCoreProc w/
# LLM, since we need a if __name__ == "__main__" guard.
context = multiprocessing.get_context("spawn")
context = get_mp_context()
process_kwargs = {
"input_path": input_path,
+9 -2
View File
@@ -159,10 +159,16 @@ class MPClient(EngineCoreClient):
atexit.register(self.shutdown)
def shutdown(self):
# During final garbage collection in process shutdown, atexit may be
# None.
if atexit:
# in case shutdown gets called via __del__ first
atexit.unregister(self.shutdown)
# Shut down the zmq context.
self.ctx.destroy(linger=0)
if hasattr(self, "proc_handle"):
if hasattr(self, "proc_handle") and self.proc_handle:
# Shutdown the process if needed.
if self.proc_handle.proc.is_alive():
self.proc_handle.proc.terminate()
@@ -178,8 +184,9 @@ class MPClient(EngineCoreClient):
]
for ipc_socket in ipc_sockets:
socket_file = ipc_socket.replace("ipc://", "")
if os.path.exists(socket_file):
if os and os.path.exists(socket_file):
os.remove(socket_file)
self.proc_handle = None
def __del__(self):
self.shutdown()
+7
View File
@@ -196,3 +196,10 @@ class LLMEngine:
f"found type: {type(tokenizer_group)}")
return tokenizer_group
def __del__(self):
self.shutdown()
def shutdown(self):
if engine_core := getattr(self, "engine_core", None):
engine_core.shutdown()