[Core][Bugfix][Perf] Introduce MQLLMEngine to avoid asyncio OH (#8157)
Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
co-authored by
Nick Hill
rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw
Simon Mo
parent
9d104b5beb
commit
7c7714d856
@@ -1,21 +1,21 @@
|
||||
import asyncio
|
||||
import signal
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Request, Response
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.async_llm_engine import AsyncEngineDeadError
|
||||
from vllm.engine.multiprocessing import MQEngineDeadError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import find_process_using_port
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
|
||||
**uvicorn_kwargs: Any):
|
||||
async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
|
||||
logger.info("Available routes are:")
|
||||
for route in app.routes:
|
||||
methods = getattr(route, "methods", None)
|
||||
@@ -26,15 +26,6 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
|
||||
|
||||
logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
|
||||
|
||||
# Set concurrency limits in uvicorn if running in multiprocessing mode
|
||||
# since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
|
||||
if limit_concurrency is not None:
|
||||
logger.info(
|
||||
"Launching Uvicorn with --limit_concurrency %s. To avoid this "
|
||||
"limit at the expense of performance run with "
|
||||
"--disable-frontend-multiprocessing", limit_concurrency)
|
||||
uvicorn_kwargs["limit_concurrency"] = limit_concurrency
|
||||
|
||||
config = uvicorn.Config(app, **uvicorn_kwargs)
|
||||
server = uvicorn.Server(config)
|
||||
_add_shutdown_handlers(app, server)
|
||||
@@ -63,7 +54,7 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
|
||||
logger.debug(
|
||||
"port %s is used by process %s launched with command:\n%s",
|
||||
port, process, " ".join(process.cmdline()))
|
||||
logger.info("Gracefully stopping http server")
|
||||
logger.info("Shutting down FastAPI HTTP server.")
|
||||
return server.shutdown()
|
||||
|
||||
|
||||
@@ -90,7 +81,7 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
|
||||
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
@app.exception_handler(AsyncEngineDeadError)
|
||||
async def engine_dead_handler(_, __):
|
||||
async def async_engine_dead_handler(_, __):
|
||||
"""Kill the server if the async engine is already dead. It will
|
||||
not handle any further requests."""
|
||||
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
|
||||
@@ -99,3 +90,14 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
|
||||
server.should_exit = True
|
||||
|
||||
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
@app.exception_handler(MQEngineDeadError)
|
||||
async def mq_engine_dead_handler(_, __):
|
||||
"""Kill the server if the mq engine is already dead. It will
|
||||
not handle any further requests."""
|
||||
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
|
||||
logger.fatal("MQLLMEngine is already dead, terminating server "
|
||||
"process")
|
||||
server.should_exit = True
|
||||
|
||||
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
@@ -26,7 +26,9 @@ import vllm.envs as envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.multiprocessing.client import MQLLMEngineClient
|
||||
from vllm.engine.multiprocessing.engine import run_mp_engine
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launcher import serve_http
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
@@ -44,8 +46,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
|
||||
TokenizeRequest,
|
||||
TokenizeResponse,
|
||||
UnloadLoraAdapterRequest)
|
||||
from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
|
||||
from vllm.entrypoints.openai.rpc.server import run_rpc_server
|
||||
# yapf: enable
|
||||
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
||||
@@ -67,29 +67,16 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
|
||||
_running_tasks: Set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def model_is_embedding(model_name: str, trust_remote_code: bool,
|
||||
quantization: Optional[str],
|
||||
revision: Optional[str]) -> bool:
|
||||
return ModelConfig(model=model_name,
|
||||
revision=revision,
|
||||
tokenizer=model_name,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=trust_remote_code,
|
||||
quantization=quantization,
|
||||
seed=0,
|
||||
dtype="auto").embedding_mode
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
if app.state.log_stats:
|
||||
async_engine_client = app.state.engine_client
|
||||
engine_client: EngineClient = app.state.engine_client
|
||||
|
||||
async def _force_log():
|
||||
while True:
|
||||
await asyncio.sleep(10)
|
||||
await async_engine_client.do_log_stats()
|
||||
await asyncio.sleep(10.)
|
||||
await engine_client.do_log_stats()
|
||||
|
||||
task = asyncio.create_task(_force_log())
|
||||
_running_tasks.add(task)
|
||||
@@ -108,9 +95,9 @@ async def lifespan(app: FastAPI):
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client(
|
||||
args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
|
||||
args: Namespace) -> AsyncIterator[Optional[EngineClient]]:
|
||||
|
||||
# Context manager to handle async_engine_client lifecycle
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
|
||||
@@ -123,19 +110,18 @@ async def build_async_engine_client(
|
||||
async def build_async_engine_client_from_engine_args(
|
||||
engine_args: AsyncEngineArgs,
|
||||
disable_frontend_multiprocessing: bool = False,
|
||||
) -> AsyncIterator[Optional[AsyncEngineClient]]:
|
||||
) -> AsyncIterator[Optional[EngineClient]]:
|
||||
"""
|
||||
Create AsyncEngineClient, either:
|
||||
Create EngineClient, either:
|
||||
- in-process using the AsyncLLMEngine Directly
|
||||
- multiprocess using AsyncLLMEngine RPC
|
||||
|
||||
Returns the Client or None if the creation failed.
|
||||
"""
|
||||
|
||||
# If manually triggered or embedding model, use AsyncLLMEngine in process.
|
||||
# TODO: support embedding model via RPC.
|
||||
if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
|
||||
engine_args.quantization, engine_args.revision)
|
||||
# Fall back
|
||||
# TODO: fill out feature matrix.
|
||||
if (MQLLMEngineClient.is_unsupported_config(engine_args)
|
||||
or disable_frontend_multiprocessing):
|
||||
engine_config = engine_args.create_engine_config()
|
||||
uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
|
||||
@@ -173,56 +159,60 @@ async def build_async_engine_client_from_engine_args(
|
||||
"and vLLM will properly handle cleanup.")
|
||||
|
||||
# Select random path for IPC.
|
||||
rpc_path = get_open_zmq_ipc_path()
|
||||
logger.info("Multiprocessing frontend to use %s for RPC Path.",
|
||||
rpc_path)
|
||||
ipc_path = get_open_zmq_ipc_path()
|
||||
logger.info("Multiprocessing frontend to use %s for IPC Path.",
|
||||
ipc_path)
|
||||
|
||||
# Build RPCClient, which conforms to AsyncEngineClient Protocol.
|
||||
# NOTE: Actually, this is not true yet. We still need to support
|
||||
# embedding models via RPC (see TODO above)
|
||||
rpc_client = AsyncEngineRPCClient(rpc_path)
|
||||
|
||||
# Start RPCServer in separate process (holds the AsyncLLMEngine).
|
||||
context = multiprocessing.get_context("spawn")
|
||||
# Start RPCServer in separate process (holds the LLMEngine).
|
||||
# the current process might have CUDA context,
|
||||
# so we need to spawn a new process
|
||||
rpc_server_process = context.Process(
|
||||
target=run_rpc_server,
|
||||
args=(engine_args, UsageContext.OPENAI_API_SERVER, rpc_path))
|
||||
rpc_server_process.start()
|
||||
logger.info("Started engine process with PID %d",
|
||||
rpc_server_process.pid)
|
||||
context = multiprocessing.get_context("spawn")
|
||||
|
||||
engine_process = context.Process(target=run_mp_engine,
|
||||
args=(engine_args,
|
||||
UsageContext.OPENAI_API_SERVER,
|
||||
ipc_path))
|
||||
engine_process.start()
|
||||
logger.info("Started engine process with PID %d", engine_process.pid)
|
||||
|
||||
# Build RPCClient, which conforms to EngineClient Protocol.
|
||||
# NOTE: Actually, this is not true yet. We still need to support
|
||||
# embedding models via RPC (see TODO above)
|
||||
engine_config = engine_args.create_engine_config()
|
||||
mp_engine_client = MQLLMEngineClient(ipc_path, engine_config)
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
await rpc_client.setup()
|
||||
await mp_engine_client.setup()
|
||||
break
|
||||
except TimeoutError:
|
||||
if not rpc_server_process.is_alive():
|
||||
logger.error(
|
||||
"RPCServer process died before responding "
|
||||
"to readiness probe")
|
||||
if not engine_process.is_alive():
|
||||
logger.error("Engine process died before responding "
|
||||
"to readiness probe")
|
||||
yield None
|
||||
return
|
||||
|
||||
yield rpc_client # type: ignore[misc]
|
||||
yield mp_engine_client # type: ignore[misc]
|
||||
finally:
|
||||
# Ensure rpc server process was terminated
|
||||
rpc_server_process.terminate()
|
||||
engine_process.terminate()
|
||||
|
||||
# Close all open connections to the backend
|
||||
rpc_client.close()
|
||||
mp_engine_client.close()
|
||||
|
||||
# Wait for server process to join
|
||||
rpc_server_process.join()
|
||||
# Wait for engine process to join
|
||||
engine_process.join(4)
|
||||
if engine_process.exitcode is None:
|
||||
# Kill if taking longer than 5 seconds to stop
|
||||
engine_process.kill()
|
||||
|
||||
# Lazy import for prometheus multiprocessing.
|
||||
# We need to set PROMETHEUS_MULTIPROC_DIR environment variable
|
||||
# before prometheus_client is imported.
|
||||
# See https://prometheus.github.io/client_python/multiprocess/
|
||||
from prometheus_client import multiprocess
|
||||
multiprocess.mark_process_dead(rpc_server_process.pid)
|
||||
multiprocess.mark_process_dead(engine_process.pid)
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
@@ -270,7 +260,7 @@ def embedding(request: Request) -> OpenAIServingEmbedding:
|
||||
return request.app.state.openai_serving_embedding
|
||||
|
||||
|
||||
def engine_client(request: Request) -> AsyncEngineClient:
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
@@ -473,7 +463,7 @@ def build_app(args: Namespace) -> FastAPI:
|
||||
|
||||
|
||||
def init_app_state(
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
@@ -488,11 +478,11 @@ def init_app_state(
|
||||
else:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
|
||||
state.engine_client = async_engine_client
|
||||
state.engine_client = engine_client
|
||||
state.log_stats = not args.disable_log_stats
|
||||
|
||||
state.openai_serving_chat = OpenAIServingChat(
|
||||
async_engine_client,
|
||||
engine_client,
|
||||
model_config,
|
||||
served_model_names,
|
||||
args.response_role,
|
||||
@@ -504,7 +494,7 @@ def init_app_state(
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser)
|
||||
state.openai_serving_completion = OpenAIServingCompletion(
|
||||
async_engine_client,
|
||||
engine_client,
|
||||
model_config,
|
||||
served_model_names,
|
||||
lora_modules=args.lora_modules,
|
||||
@@ -513,13 +503,13 @@ def init_app_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
)
|
||||
state.openai_serving_embedding = OpenAIServingEmbedding(
|
||||
async_engine_client,
|
||||
engine_client,
|
||||
model_config,
|
||||
served_model_names,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
state.openai_serving_tokenization = OpenAIServingTokenization(
|
||||
async_engine_client,
|
||||
engine_client,
|
||||
model_config,
|
||||
served_model_names,
|
||||
lora_modules=args.lora_modules,
|
||||
@@ -541,21 +531,20 @@ async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
async with build_async_engine_client(args) as async_engine_client:
|
||||
async with build_async_engine_client(args) as engine_client:
|
||||
# If None, creation of the client failed and we exit.
|
||||
if async_engine_client is None:
|
||||
if engine_client is None:
|
||||
return
|
||||
|
||||
app = build_app(args)
|
||||
|
||||
model_config = await async_engine_client.get_model_config()
|
||||
init_app_state(async_engine_client, model_config, app.state, args)
|
||||
model_config = await engine_client.get_model_config()
|
||||
init_app_state(engine_client, model_config, app.state, args)
|
||||
|
||||
temp_socket.close()
|
||||
|
||||
shutdown_task = await serve_http(
|
||||
app,
|
||||
limit_concurrency=async_engine_client.limit_concurrency,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Mapping, Optional, Union
|
||||
|
||||
from vllm.inputs import PromptInputs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
# Success string used for RPC instructions.
|
||||
VLLM_RPC_SUCCESS_STR = "SUCCESS"
|
||||
|
||||
# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
|
||||
VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
|
||||
|
||||
# HWM is set to Infinity.
|
||||
VLLM_RPC_ZMQ_HWM = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class RPCGenerateRequest:
|
||||
inputs: PromptInputs
|
||||
sampling_params: SamplingParams
|
||||
request_id: str
|
||||
lora_request: Optional[LoRARequest] = None
|
||||
trace_headers: Optional[Mapping[str, str]] = None
|
||||
prompt_adapter_request: Optional[PromptAdapterRequest] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RPCAbortRequest:
|
||||
request_id: str
|
||||
|
||||
|
||||
class RPCUtilityRequest(Enum):
|
||||
IS_SERVER_READY = 1
|
||||
GET_MODEL_CONFIG = 2
|
||||
GET_DECODING_CONFIG = 3
|
||||
GET_PARALLEL_CONFIG = 4
|
||||
GET_SCHEDULER_CONFIG = 5
|
||||
GET_LORA_CONFIG = 6
|
||||
DO_LOG_STATS = 7
|
||||
IS_SERVER_HEALTHY = 8
|
||||
IS_TRACING_ENABLED = 9
|
||||
START_PROFILE = 10
|
||||
STOP_PROFILE = 11
|
||||
|
||||
|
||||
RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
|
||||
RPCUtilityRequest]
|
||||
@@ -1,451 +0,0 @@
|
||||
import asyncio
|
||||
import pickle
|
||||
from contextlib import contextmanager, suppress
|
||||
from typing import Any, AsyncGenerator, Iterator, Mapping, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
import cloudpickle
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
from zmq import Frame # type: ignore[attr-defined]
|
||||
from zmq.asyncio import Socket
|
||||
|
||||
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig)
|
||||
# yapf: disable
|
||||
from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
|
||||
VLLM_RPC_SOCKET_LIMIT_CUTOFF,
|
||||
VLLM_RPC_SUCCESS_STR,
|
||||
VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
|
||||
RPCGenerateRequest, RPCUtilityRequest)
|
||||
# yapf: enable
|
||||
from vllm.envs import VLLM_RPC_GET_DATA_TIMEOUT_MS
|
||||
from vllm.inputs import PromptInputs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Path used for inprocess proxy.
|
||||
INPROC_PROXY_PATH = f"inproc://{uuid4()}"
|
||||
|
||||
|
||||
class RPCClientClosedError(Exception):
|
||||
"""Exception class raised when the client is used post-close.
|
||||
|
||||
The client can be closed, which closes the ZMQ context. This normally
|
||||
happens on server shutdown. In some cases, methods like abort and
|
||||
do_log_stats will still be called and then try to open a socket, which
|
||||
causes a ZMQError and creates a huge stack trace.
|
||||
So, we throw this error such that we can suppress it.
|
||||
"""
|
||||
|
||||
|
||||
class AsyncEngineRPCClient:
|
||||
"""
|
||||
RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
|
||||
|
||||
The overall design mirrors the Asynchronous Client Server Pattern
|
||||
https://zguide.zeromq.org/docs/chapter3/#The-Asynchronous-Client-Server-Pattern
|
||||
|
||||
On startup, the RPCClient:
|
||||
- makes DEALER socket (to_rpc_server) that connects to the RPCServer
|
||||
via ipc, which uses unix sockets under the hood
|
||||
(https://libzmq.readthedocs.io/en/zeromq4-1/zmq_ipc.html)
|
||||
- makes ROUTER socket (from_api_server) that binds to a random
|
||||
inproc address, which uses memory under the hood
|
||||
(https://libzmq.readthedocs.io/en/zeromq3-x/zmq_inproc.html)
|
||||
- runs a proxy in a background asyncio task between
|
||||
from_api_server (ROUTER, inproc) and to_rpc_server (DEALER ipc, )
|
||||
|
||||
Each request handled by the asyncio api_server calls generate():
|
||||
- make a DEALER socket that connects to from_api_server via inproc
|
||||
- send a RCPGenerateRequest to the inproc socket
|
||||
- background proxy forwards the request from inproc -> ipc
|
||||
- RPCServer responds to the request one token at a time over ipc
|
||||
- background proxy forwards the response from ipc -> inproc
|
||||
|
||||
The connection looks like this:
|
||||
DEALER <- inproc -> [ ROUTER | DEALER ] <- ipc -> DEALER
|
||||
|
||||
Message routing is performed via identities that are managed by the
|
||||
ROUTER socket. ROUTER sockets track every connection it has and
|
||||
tells the caller about these. The way it tells the caller is to stick
|
||||
the connection identity in front of each message received. When we
|
||||
send the message via a ROUTER, we first send an identity frame.
|
||||
See https://zguide.zeromq.org/docs/chapter3/#The-Extended-Reply-Envelope
|
||||
for more details on connection identities.
|
||||
|
||||
This proxy design enables us to use a single unix socket, which
|
||||
improves performance by avoiding syscalls (~5%) and avoids resource limits
|
||||
such as ulimit, which defaults to 1024 on ubuntu.
|
||||
|
||||
Note: we run set_hwm(0) on each socket, which sets the HWM to inf,
|
||||
which is required to avoid dropping messages under high load.
|
||||
This is generally not advisable. However, since we are in control
|
||||
of both sides of the connection + failure on either side is
|
||||
catastrophic to the overall system health and memory profiling
|
||||
suggests limited memory overhead relative to asyncio, we will
|
||||
proceed for now.
|
||||
|
||||
See https://zguide.zeromq.org/docs/chapter2/#High-Water-Marks
|
||||
for more details on high water marks.
|
||||
"""
|
||||
|
||||
def __init__(self, rpc_path: str):
|
||||
self.context = zmq.asyncio.Context()
|
||||
self._data_timeout = VLLM_RPC_GET_DATA_TIMEOUT_MS
|
||||
self._errored = False
|
||||
|
||||
# Maximum number of sockets that can be opened (typically 65536).
|
||||
# ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
|
||||
socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
|
||||
assert isinstance(socket_limit, int)
|
||||
if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
|
||||
raise ValueError(
|
||||
f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
|
||||
"the number of concurrent requests vLLM can process. Launch "
|
||||
"vLLM with --disable-frontend-multiprocessing and open a "
|
||||
"GitHub issue so we can investigate.")
|
||||
|
||||
# We only have 1 ipc connection that uses unix sockets, so
|
||||
# safe to set MAX_SOCKETS to the zmq SOCKET_LIMIT (i.e. will
|
||||
# not run into ulimit issues)
|
||||
self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
|
||||
|
||||
# IPC connection to RPC Server (uses unix sockets).
|
||||
self.to_rpc_server: Socket = self.context.socket(zmq.constants.DEALER)
|
||||
self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
|
||||
self.to_rpc_server.bind(rpc_path)
|
||||
|
||||
# In process proxy to RPC Server (uses memory-based messaging).
|
||||
self.from_api_server: Socket = self.context.socket(
|
||||
zmq.constants.ROUTER)
|
||||
self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
|
||||
self.from_api_server.bind(INPROC_PROXY_PATH)
|
||||
|
||||
# Asyncio background task for the proxy.
|
||||
self.proxy_in_task = asyncio.create_task(
|
||||
self.run_proxy(self.from_api_server, self.to_rpc_server))
|
||||
self.proxy_out_task = asyncio.create_task(
|
||||
self.run_proxy(self.to_rpc_server, self.from_api_server))
|
||||
|
||||
# Since we open 1 inproc socket per request, we have a hard cap on
|
||||
# the number of requests that can run in vLLM w. frontend
|
||||
# mulitprocessing. This value is used uvicorn to launch
|
||||
# with --limit-concurrency to return 503 when server is overloaded.
|
||||
# We need 2 sockets per request - 2:
|
||||
# 1 for generate(), 1 for abort(), do_log_stats(), check_health()
|
||||
self.limit_concurrency = socket_limit // 2 - 2
|
||||
|
||||
async def run_proxy(self, socket_from: Socket, socket_to: Socket):
|
||||
"""Background task that runs a proxy"""
|
||||
while True:
|
||||
frames = await socket_from.recv_multipart(copy=False)
|
||||
await socket_to.send_multipart(frames, copy=False)
|
||||
|
||||
async def setup(self):
|
||||
"""Setup the client before it starts sending server requests."""
|
||||
|
||||
# Wait until server is ready.
|
||||
await self._wait_for_server_rpc()
|
||||
|
||||
# Get the configs.
|
||||
self.model_config = await self._get_model_config_rpc()
|
||||
self.decoding_config = await self._get_decoding_config_rpc()
|
||||
self.tracing_flag = await self._is_tracing_enabled_rpc()
|
||||
|
||||
# Create the tokenizer group.
|
||||
# TODO: refactor OAI server to avoid needing this info.
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
model_config=self.model_config,
|
||||
scheduler_config=(await self._get_scheduler_config_rpc()),
|
||||
parallel_config=(await self._get_parallel_config_rpc()),
|
||||
enable_lora=bool(await self._get_lora_config_rpc()),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
"""Destroy the ZeroMQ Context."""
|
||||
# Close all sockets associated with this context and
|
||||
# then terminate the context.
|
||||
self.from_api_server.close()
|
||||
self.to_rpc_server.close()
|
||||
self.context.destroy()
|
||||
|
||||
@contextmanager
|
||||
def to_proxy_socket(self) -> Iterator[Socket]:
|
||||
# Connect to the RPCServer via the proxy.
|
||||
|
||||
# Raise a sensible error if the client was already closed.
|
||||
# This can happen if a server shutdown is triggered but some coroutines
|
||||
# are still running requests.
|
||||
# There should not be a race condition with this check because we don't
|
||||
# yield to the event loop between here and opening the socket.
|
||||
if self.context.closed:
|
||||
raise RPCClientClosedError("The ZMQ client has already shut down")
|
||||
|
||||
# Note that we use DEALER to enable asynchronous communication
|
||||
# to enable streaming.
|
||||
socket = self.context.socket(zmq.constants.DEALER)
|
||||
socket.set_hwm(VLLM_RPC_ZMQ_HWM)
|
||||
try:
|
||||
socket.connect(INPROC_PROXY_PATH)
|
||||
yield socket
|
||||
finally:
|
||||
socket.close(linger=0)
|
||||
|
||||
async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
|
||||
expected_type: Any,
|
||||
error_message: str) -> Any:
|
||||
"""Send an RPC request that is expecting data back."""
|
||||
|
||||
with self.to_proxy_socket() as socket:
|
||||
# Ping RPCServer with a request.
|
||||
await socket.send_multipart((cloudpickle.dumps(request), ),
|
||||
copy=False)
|
||||
|
||||
# Make sure the server responds
|
||||
if await socket.poll(timeout=self._data_timeout) == 0:
|
||||
raise TimeoutError("Server didn't reply within "
|
||||
f"{self._data_timeout} ms")
|
||||
|
||||
# Await the data from the Server.
|
||||
frame = await socket.recv(copy=False)
|
||||
assert isinstance(frame, Frame)
|
||||
data = pickle.loads(frame.buffer)
|
||||
|
||||
if isinstance(data, Exception):
|
||||
# Re-raise exceptions returned by the server
|
||||
raise data
|
||||
|
||||
if not isinstance(data, expected_type):
|
||||
# LoRAConfig can be None.
|
||||
if expected_type == LoRAConfig and data is None:
|
||||
pass
|
||||
elif isinstance(data, Exception):
|
||||
logger.error(error_message)
|
||||
raise data
|
||||
else:
|
||||
raise ValueError(error_message)
|
||||
|
||||
return data
|
||||
|
||||
async def _send_one_way_rpc_request(self,
|
||||
request: RPC_REQUEST_TYPE,
|
||||
error_message: str,
|
||||
socket: Optional[Socket] = None):
|
||||
"""Send one-way RPC request to trigger an action."""
|
||||
|
||||
async def do_rpc_call(socket: Socket, request: RPC_REQUEST_TYPE):
|
||||
|
||||
await socket.send_multipart((cloudpickle.dumps(request), ))
|
||||
|
||||
if await socket.poll(timeout=self._data_timeout) == 0:
|
||||
raise TimeoutError("Server didn't reply within "
|
||||
f"{self._data_timeout} ms")
|
||||
|
||||
frame = await socket.recv(copy=False)
|
||||
assert isinstance(frame, Frame)
|
||||
return pickle.loads(frame.buffer)
|
||||
|
||||
# Make a new socket connection.
|
||||
if socket is None:
|
||||
with self.to_proxy_socket() as socket:
|
||||
response = await do_rpc_call(socket, request)
|
||||
|
||||
# Use existing socket connection.
|
||||
else:
|
||||
response = await do_rpc_call(socket, request)
|
||||
|
||||
if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
|
||||
if isinstance(response, Exception):
|
||||
logger.error(error_message)
|
||||
raise response
|
||||
raise ValueError(error_message)
|
||||
|
||||
async def get_tokenizer(self, lora_request: LoRARequest):
|
||||
return await self.tokenizer.get_lora_tokenizer_async(lora_request)
|
||||
|
||||
async def get_decoding_config(self) -> DecodingConfig:
|
||||
return self.decoding_config
|
||||
|
||||
async def get_model_config(self) -> ModelConfig:
|
||||
return self.model_config
|
||||
|
||||
async def is_tracing_enabled(self) -> bool:
|
||||
return self.tracing_flag
|
||||
|
||||
async def _wait_for_server_rpc(self):
|
||||
"""Wait for the RPCServer to start up."""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUtilityRequest.IS_SERVER_READY,
|
||||
error_message="Unable to start RPC Server")
|
||||
|
||||
async def _get_model_config_rpc(self) -> ModelConfig:
|
||||
"""Get the ModelConfig object from the RPC Server"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.GET_MODEL_CONFIG,
|
||||
expected_type=ModelConfig,
|
||||
error_message="Could not get ModelConfig from RPC Server")
|
||||
|
||||
async def _get_decoding_config_rpc(self) -> DecodingConfig:
|
||||
"""Get DecodingConfig from the RPCServer"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.GET_DECODING_CONFIG,
|
||||
expected_type=DecodingConfig,
|
||||
error_message="Could not get DecodingConfig from RPC Server")
|
||||
|
||||
async def _get_parallel_config_rpc(self) -> ParallelConfig:
|
||||
"""Get ParallelConfig from the RPCServer"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.GET_PARALLEL_CONFIG,
|
||||
expected_type=ParallelConfig,
|
||||
error_message="Could not get ParallelConfig from RPC Server")
|
||||
|
||||
async def _get_scheduler_config_rpc(self) -> SchedulerConfig:
|
||||
"""Get SchedulerConfig from the RPCServer"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.GET_SCHEDULER_CONFIG,
|
||||
expected_type=SchedulerConfig,
|
||||
error_message="Could not get SchedulerConfig from RPC Server")
|
||||
|
||||
async def _get_lora_config_rpc(self) -> LoRAConfig:
|
||||
"""Get LoRAConfig from the RPCServer"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.GET_LORA_CONFIG,
|
||||
expected_type=LoRAConfig,
|
||||
error_message="Could not get LoRAConfig from RPC Server")
|
||||
|
||||
async def _is_tracing_enabled_rpc(self) -> bool:
|
||||
"""Get is_tracing_enabled flag from the RPCServer"""
|
||||
|
||||
return await self._send_get_data_rpc_request(
|
||||
RPCUtilityRequest.IS_TRACING_ENABLED,
|
||||
expected_type=bool,
|
||||
error_message="Could not get is_tracing_enabled from RPC Server")
|
||||
|
||||
async def abort(self, request_id: str):
|
||||
"""Send an ABORT_REQUEST signal to the RPC Server"""
|
||||
|
||||
# Suppress timeouts as well.
|
||||
# In cases where the server is busy processing requests and a very
|
||||
# large volume of abort requests arrive, it is likely that the server
|
||||
# will not be able to ack all of them in time. We have seen this when
|
||||
# we abort 20k requests at once while another 2k are processing- many
|
||||
# of them time out, but we see the server successfully abort all of the
|
||||
# requests.
|
||||
# In this case we assume that the server has received or will receive
|
||||
# these abort requests, and ignore the timeout. This prevents a massive
|
||||
# wall of `TimeoutError` stack traces.
|
||||
with suppress(RPCClientClosedError, TimeoutError):
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCAbortRequest(request_id),
|
||||
error_message=f"RPCAbortRequest {request_id} failed")
|
||||
|
||||
async def do_log_stats(self):
|
||||
"""Send a DO_LOG_STATS signal to the RPC Server"""
|
||||
with suppress(RPCClientClosedError):
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUtilityRequest.DO_LOG_STATS,
|
||||
error_message="RPCRequest DO_LOG_STATS failed.")
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return not self._errored
|
||||
|
||||
@property
|
||||
def is_stopped(self) -> bool:
|
||||
return self._errored
|
||||
|
||||
@property
|
||||
def errored(self) -> bool:
|
||||
return self._errored
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
inputs: PromptInputs,
|
||||
sampling_params: SamplingParams,
|
||||
request_id: str,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt_adapter_request: Optional[PromptAdapterRequest] = None
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""Send an RPCGenerateRequest to the RPCServer and stream responses."""
|
||||
|
||||
finished = False
|
||||
try:
|
||||
with self.to_proxy_socket() as socket:
|
||||
# Send RPCGenerateRequest to the RPCServer.
|
||||
await socket.send_multipart((cloudpickle.dumps(
|
||||
RPCGenerateRequest(
|
||||
inputs=inputs,
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
prompt_adapter_request=prompt_adapter_request)), ))
|
||||
|
||||
# Stream back the results from the RPC Server.
|
||||
while not finished:
|
||||
message = await socket.recv(copy=False)
|
||||
assert isinstance(message, Frame)
|
||||
request_output = pickle.loads(message.buffer)
|
||||
|
||||
if isinstance(request_output, Exception):
|
||||
# On exception, check if the server is still healthy
|
||||
# possibly setting the `errored` property.
|
||||
if not self._errored:
|
||||
try:
|
||||
await self.check_health(socket=socket)
|
||||
except Exception as e:
|
||||
self._errored = True
|
||||
logger.exception(repr(e))
|
||||
|
||||
# NB: do before raising here so that the flag is set
|
||||
# by the time the caller receives this exception
|
||||
raise request_output
|
||||
|
||||
finished = request_output.finished
|
||||
yield request_output
|
||||
|
||||
finally:
|
||||
# Request was canceled by the client.
|
||||
if not finished and not self._errored:
|
||||
await self.abort(request_id)
|
||||
|
||||
async def check_health(self, socket: Optional[Socket] = None) -> None:
|
||||
"""Raise if unhealthy"""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUtilityRequest.IS_SERVER_HEALTHY,
|
||||
error_message="Got Unhealthy response from RPC Server",
|
||||
socket=socket)
|
||||
|
||||
async def encode(self, *args,
|
||||
**kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
|
||||
raise NotImplementedError(
|
||||
"Embeddings not supported with multiprocessing backend")
|
||||
|
||||
async def start_profile(self) -> None:
|
||||
"""Start profiling the engine"""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUtilityRequest.START_PROFILE,
|
||||
error_message="RPCRequest START_PROFILE failed.")
|
||||
|
||||
async def stop_profile(self) -> None:
|
||||
"""Stop profiling the engine"""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUtilityRequest.STOP_PROFILE,
|
||||
error_message="RPCRequest STOP_PROFILE failed.")
|
||||
@@ -1,243 +0,0 @@
|
||||
import asyncio
|
||||
import pickle
|
||||
import signal
|
||||
from typing import Any, Coroutine, Union
|
||||
|
||||
import cloudpickle
|
||||
import uvloop
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
from typing_extensions import Never
|
||||
from zmq import Frame # type: ignore[attr-defined]
|
||||
from zmq.asyncio import Socket
|
||||
|
||||
from vllm import AsyncEngineArgs, AsyncLLMEngine
|
||||
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig)
|
||||
from vllm.entrypoints.openai.rpc import (VLLM_RPC_SUCCESS_STR,
|
||||
VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
|
||||
RPCGenerateRequest, RPCUtilityRequest)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
|
||||
SchedulerConfig, LoRAConfig]
|
||||
|
||||
|
||||
class AsyncEngineRPCServer:
|
||||
|
||||
def __init__(self, async_engine_args: AsyncEngineArgs,
|
||||
usage_context: UsageContext, rpc_path: str):
|
||||
# Initialize engine first.
|
||||
self.engine = AsyncLLMEngine.from_engine_args(
|
||||
async_engine_args, usage_context=usage_context)
|
||||
|
||||
# Initialize context.
|
||||
self.context = zmq.asyncio.Context()
|
||||
|
||||
# Init socket.
|
||||
self.socket: Socket = self.context.socket(zmq.constants.DEALER)
|
||||
self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
|
||||
self.socket.connect(rpc_path)
|
||||
|
||||
def cleanup(self):
|
||||
"""Cleanup all resources."""
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
# Clear the engine reference so that it can be GC'ed.
|
||||
del self.engine
|
||||
|
||||
async def get_config(self, identity, request):
|
||||
try:
|
||||
config: CONFIG_TYPE
|
||||
if request == RPCUtilityRequest.GET_MODEL_CONFIG:
|
||||
config = await self.engine.get_model_config()
|
||||
elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
|
||||
config = await self.engine.get_decoding_config()
|
||||
elif request == RPCUtilityRequest.GET_LORA_CONFIG:
|
||||
config = await self.engine.get_lora_config()
|
||||
elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
|
||||
config = await self.engine.get_scheduler_config()
|
||||
elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
|
||||
config = await self.engine.get_parallel_config()
|
||||
else:
|
||||
raise ValueError("Unknown Config Request: %s", request)
|
||||
|
||||
await self.socket.send_multipart((identity, pickle.dumps(config)),
|
||||
copy=False)
|
||||
|
||||
except Exception as e:
|
||||
await self.socket.send_multipart((identity, pickle.dumps(e)),
|
||||
copy=False)
|
||||
|
||||
async def is_tracing_enabled(self, identity):
|
||||
"""Send the is_tracing_enabled flag"""
|
||||
tracing_flag = await self.engine.is_tracing_enabled()
|
||||
|
||||
await self.socket.send_multipart(
|
||||
(identity, pickle.dumps(tracing_flag)))
|
||||
|
||||
async def do_log_stats(self, identity):
|
||||
"""Log stats and confirm success."""
|
||||
await self.engine.do_log_stats()
|
||||
|
||||
await self.socket.send_multipart(
|
||||
(identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
|
||||
|
||||
async def is_server_ready(self, identity):
|
||||
"""Notify the client that we are ready."""
|
||||
await self.socket.send_multipart(
|
||||
(identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
|
||||
|
||||
async def abort(self, identity, request: RPCAbortRequest):
|
||||
"""Abort request and notify the client of success."""
|
||||
try:
|
||||
# Abort the request in the llm engine.
|
||||
await self.engine.abort(request.request_id)
|
||||
result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
|
||||
except Exception as e:
|
||||
result = e
|
||||
await self.socket.send_multipart((identity, pickle.dumps(result)))
|
||||
|
||||
async def generate(self, identity, generate_request: RPCGenerateRequest):
|
||||
try:
|
||||
results_generator = self.engine.generate(
|
||||
generate_request.inputs,
|
||||
sampling_params=generate_request.sampling_params,
|
||||
request_id=generate_request.request_id,
|
||||
lora_request=generate_request.lora_request,
|
||||
trace_headers=generate_request.trace_headers,
|
||||
prompt_adapter_request=generate_request.prompt_adapter_request)
|
||||
|
||||
async for request_output in results_generator:
|
||||
await self.socket.send_multipart(
|
||||
(identity, pickle.dumps(request_output)), copy=False)
|
||||
|
||||
except Exception as e:
|
||||
await self.socket.send_multipart((identity, pickle.dumps(e)),
|
||||
copy=False)
|
||||
|
||||
async def check_health(self, identity):
|
||||
try:
|
||||
await self.engine.check_health()
|
||||
await self.socket.send_multipart(
|
||||
(identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
|
||||
|
||||
except Exception as e:
|
||||
await self.socket.send_multipart((identity, pickle.dumps(e)),
|
||||
copy=False)
|
||||
|
||||
async def start_profile(self, identity):
|
||||
logger.info("Starting profiler...")
|
||||
await self.engine.start_profile()
|
||||
logger.info("Profiler started.")
|
||||
|
||||
await self.socket.send_multipart((
|
||||
identity,
|
||||
pickle.dumps(VLLM_RPC_SUCCESS_STR),
|
||||
))
|
||||
|
||||
async def stop_profile(self, identity):
|
||||
logger.info("Stopping profiler...")
|
||||
await self.engine.stop_profile()
|
||||
logger.info("Profiler stopped.")
|
||||
|
||||
await self.socket.send_multipart((
|
||||
identity,
|
||||
pickle.dumps(VLLM_RPC_SUCCESS_STR),
|
||||
))
|
||||
|
||||
def _make_handler_coro(self, identity,
|
||||
message: Frame) -> Coroutine[Any, Any, Never]:
|
||||
"""Route the zmq message to the handler coroutine."""
|
||||
|
||||
request = cloudpickle.loads(message.buffer)
|
||||
|
||||
if isinstance(request, RPCGenerateRequest):
|
||||
return self.generate(identity, request)
|
||||
|
||||
elif isinstance(request, RPCAbortRequest):
|
||||
return self.abort(identity, request)
|
||||
|
||||
elif isinstance(request, RPCUtilityRequest):
|
||||
if request in [
|
||||
RPCUtilityRequest.GET_MODEL_CONFIG,
|
||||
RPCUtilityRequest.GET_PARALLEL_CONFIG,
|
||||
RPCUtilityRequest.GET_DECODING_CONFIG,
|
||||
RPCUtilityRequest.GET_SCHEDULER_CONFIG,
|
||||
RPCUtilityRequest.GET_LORA_CONFIG
|
||||
]:
|
||||
return self.get_config(identity, request)
|
||||
elif request == RPCUtilityRequest.DO_LOG_STATS:
|
||||
return self.do_log_stats(identity)
|
||||
elif request == RPCUtilityRequest.IS_SERVER_READY:
|
||||
return self.is_server_ready(identity)
|
||||
elif request == RPCUtilityRequest.IS_SERVER_HEALTHY:
|
||||
return self.check_health(identity)
|
||||
elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
|
||||
return self.is_tracing_enabled(identity)
|
||||
elif request == RPCUtilityRequest.START_PROFILE:
|
||||
return self.start_profile(identity)
|
||||
elif request == RPCUtilityRequest.STOP_PROFILE:
|
||||
return self.stop_profile(identity)
|
||||
else:
|
||||
raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown RPCRequest type: {request}")
|
||||
|
||||
async def run_server_loop(self):
|
||||
"""Inner RPC Server Loop"""
|
||||
|
||||
running_tasks = set()
|
||||
while True:
|
||||
# Wait for a request.
|
||||
identity, message = await self.socket.recv_multipart(copy=False)
|
||||
|
||||
# Process the request async.
|
||||
task = asyncio.create_task(
|
||||
self._make_handler_coro(identity, message))
|
||||
|
||||
# We need to keep around a strong reference to the task,
|
||||
# to avoid the task disappearing mid-execution as running tasks
|
||||
# can be GC'ed. Below is a common "fire-and-forget" tasks
|
||||
# https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
|
||||
running_tasks.add(task)
|
||||
task.add_done_callback(running_tasks.discard)
|
||||
|
||||
|
||||
async def run_server(server: AsyncEngineRPCServer):
|
||||
# Put the server task into the asyncio loop.
|
||||
loop = asyncio.get_running_loop()
|
||||
server_task = loop.create_task(server.run_server_loop())
|
||||
|
||||
# Interruption handling.
|
||||
def signal_handler() -> None:
|
||||
# Kill the server on interrupt / terminate
|
||||
server_task.cancel()
|
||||
|
||||
loop.add_signal_handler(signal.SIGINT, signal_handler)
|
||||
loop.add_signal_handler(signal.SIGTERM, signal_handler)
|
||||
|
||||
try:
|
||||
await server_task
|
||||
except asyncio.CancelledError:
|
||||
logger.info("vLLM ZMQ RPC Server was interrupted.")
|
||||
finally:
|
||||
# Clean up all resources.
|
||||
server.cleanup()
|
||||
|
||||
|
||||
def run_rpc_server(async_engine_args: AsyncEngineArgs,
|
||||
usage_context: UsageContext, rpc_path: str):
|
||||
|
||||
def signal_handler(*_) -> None:
|
||||
# Interrupt server on sigterm while initializing
|
||||
raise KeyboardInterrupt("AsyncEngineRPCServer terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
|
||||
uvloop.run(run_server(server))
|
||||
@@ -9,7 +9,7 @@ from typing import Union
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (ConversationMessage,
|
||||
apply_hf_chat_template,
|
||||
apply_mistral_chat_template,
|
||||
@@ -45,7 +45,7 @@ logger = init_logger(__name__)
|
||||
class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
def __init__(self,
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
served_model_names: List[str],
|
||||
response_role: str,
|
||||
@@ -57,7 +57,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: Optional[str] = None):
|
||||
super().__init__(async_engine_client=async_engine_client,
|
||||
super().__init__(engine_client=engine_client,
|
||||
model_config=model_config,
|
||||
served_model_names=served_model_names,
|
||||
lora_modules=lora_modules,
|
||||
@@ -105,6 +105,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
try:
|
||||
(
|
||||
lora_request,
|
||||
@@ -112,8 +118,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
) = self._maybe_get_adapters(request)
|
||||
|
||||
model_config = self.model_config
|
||||
tokenizer = await self.async_engine_client.get_tokenizer(
|
||||
lora_request)
|
||||
tokenizer = await self.engine_client.get_tokenizer(lora_request)
|
||||
|
||||
conversation, mm_data_future = parse_chat_messages_futures(
|
||||
request.messages, model_config, tokenizer)
|
||||
@@ -207,8 +212,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if mm_data is not None:
|
||||
engine_inputs["multi_modal_data"] = mm_data
|
||||
|
||||
is_tracing_enabled = (
|
||||
await self.async_engine_client.is_tracing_enabled())
|
||||
is_tracing_enabled = (await
|
||||
self.engine_client.is_tracing_enabled())
|
||||
trace_headers = None
|
||||
if is_tracing_enabled and raw_request:
|
||||
trace_headers = extract_trace_headers(raw_request.headers)
|
||||
@@ -216,7 +221,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and contains_trace_headers(raw_request.headers)):
|
||||
log_tracing_disabled_warning()
|
||||
|
||||
result_generator = self.async_engine_client.generate(
|
||||
result_generator = self.engine_client.generate(
|
||||
engine_inputs,
|
||||
sampling_params,
|
||||
request_id,
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Tuple, Union, cast
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
@@ -43,7 +43,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
served_model_names: List[str],
|
||||
*,
|
||||
@@ -52,7 +52,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
request_logger: Optional[RequestLogger],
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
):
|
||||
super().__init__(async_engine_client=async_engine_client,
|
||||
super().__init__(engine_client=engine_client,
|
||||
model_config=model_config,
|
||||
served_model_names=served_model_names,
|
||||
lora_modules=lora_modules,
|
||||
@@ -78,6 +78,12 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
# Return error for unsupported features.
|
||||
if request.suffix is not None:
|
||||
return self.create_error_response(
|
||||
@@ -95,8 +101,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
prompt_adapter_request,
|
||||
) = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.async_engine_client.get_tokenizer(
|
||||
lora_request)
|
||||
tokenizer = await self.engine_client.get_tokenizer(lora_request)
|
||||
|
||||
guided_decode_logits_processor = (
|
||||
await self._guided_decode_logits_processor(request, tokenizer))
|
||||
@@ -124,8 +129,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
prompt_adapter_request=prompt_adapter_request)
|
||||
|
||||
is_tracing_enabled = (
|
||||
await self.async_engine_client.is_tracing_enabled())
|
||||
is_tracing_enabled = (await
|
||||
self.engine_client.is_tracing_enabled())
|
||||
trace_headers = None
|
||||
if is_tracing_enabled:
|
||||
trace_headers = extract_trace_headers(raw_request.headers)
|
||||
@@ -133,7 +138,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
raw_request.headers):
|
||||
log_tracing_disabled_warning()
|
||||
|
||||
generator = self.async_engine_client.generate(
|
||||
generator = self.engine_client.generate(
|
||||
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi import Request
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
@@ -71,13 +71,13 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
served_model_names: List[str],
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
):
|
||||
super().__init__(async_engine_client=async_engine_client,
|
||||
super().__init__(engine_client=engine_client,
|
||||
model_config=model_config,
|
||||
served_model_names=served_model_names,
|
||||
lora_modules=None,
|
||||
@@ -118,8 +118,7 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
prompt_adapter_request,
|
||||
) = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.async_engine_client.get_tokenizer(
|
||||
lora_request)
|
||||
tokenizer = await self.engine_client.get_tokenizer(lora_request)
|
||||
|
||||
pooling_params = request.to_pooling_params()
|
||||
|
||||
@@ -144,7 +143,7 @@ class OpenAIServingEmbedding(OpenAIServing):
|
||||
"Prompt adapter is not supported "
|
||||
"for embedding models")
|
||||
|
||||
generator = self.async_engine_client.encode(
|
||||
generator = self.engine_client.encode(
|
||||
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
|
||||
pooling_params,
|
||||
request_id_item,
|
||||
|
||||
@@ -8,7 +8,7 @@ from pydantic import Field
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
@@ -64,7 +64,7 @@ class OpenAIServing:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
served_model_names: List[str],
|
||||
*,
|
||||
@@ -75,7 +75,7 @@ class OpenAIServing:
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.async_engine_client = async_engine_client
|
||||
self.engine_client = engine_client
|
||||
self.model_config = model_config
|
||||
self.max_model_len = model_config.max_model_len
|
||||
|
||||
@@ -159,7 +159,7 @@ class OpenAIServing:
|
||||
async def _guided_decode_logits_processor(
|
||||
self, request: Union[ChatCompletionRequest, CompletionRequest],
|
||||
tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
|
||||
decoding_config = await self.async_engine_client.get_decoding_config()
|
||||
decoding_config = await self.engine_client.get_decoding_config()
|
||||
guided_decoding_backend = request.guided_decoding_backend \
|
||||
or decoding_config.guided_decoding_backend
|
||||
return await get_guided_decoding_logits_processor(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
|
||||
apply_mistral_chat_template,
|
||||
load_chat_template,
|
||||
@@ -29,7 +29,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
async_engine_client: AsyncEngineClient,
|
||||
engine_client: EngineClient,
|
||||
model_config: ModelConfig,
|
||||
served_model_names: List[str],
|
||||
*,
|
||||
@@ -37,7 +37,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
):
|
||||
super().__init__(async_engine_client=async_engine_client,
|
||||
super().__init__(engine_client=engine_client,
|
||||
model_config=model_config,
|
||||
served_model_names=served_model_names,
|
||||
lora_modules=lora_modules,
|
||||
@@ -66,7 +66,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
prompt_adapter_request,
|
||||
) = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
|
||||
tokenizer = await self.engine_client.get_tokenizer(lora_request)
|
||||
|
||||
prompt: Union[str, List[int]]
|
||||
if isinstance(request, TokenizeChatRequest):
|
||||
@@ -132,7 +132,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
prompt_adapter_request,
|
||||
) = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
|
||||
tokenizer = await self.engine_client.get_tokenizer(lora_request)
|
||||
|
||||
self._log_inputs(request_id,
|
||||
request.tokens,
|
||||
|
||||
Reference in New Issue
Block a user