[Frontend] Add vllm launch command for GPU-less preprocessing serving (#34551)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
|
||||
import uvloop
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
build_and_serve,
|
||||
setup_server,
|
||||
)
|
||||
from vllm.entrypoints.openai.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DESCRIPTION = "Launch individual vLLM components."
|
||||
|
||||
|
||||
class LaunchSubcommandBase(CLISubcommand):
|
||||
"""The base class of subcommands for `vllm launch`."""
|
||||
|
||||
help: str
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser.
|
||||
|
||||
By default, adds the standard vLLM serving arguments.
|
||||
Subclasses can override to add component-specific arguments.
|
||||
"""
|
||||
make_arg_parser(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RenderSubcommand(LaunchSubcommandBase):
|
||||
"""The `render` subcommand for `vllm launch`."""
|
||||
|
||||
name = "render"
|
||||
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
|
||||
|
||||
class LaunchSubcommand(CLISubcommand):
|
||||
"""The `launch` subcommand for the vLLM CLI.
|
||||
|
||||
Uses nested sub-subcommands so each component can define its own
|
||||
arguments independently (e.g. ``vllm launch render``).
|
||||
"""
|
||||
|
||||
name = "launch"
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
if hasattr(args, "model_tag") and args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
|
||||
args.launch_command(args)
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
launch_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help=DESCRIPTION,
|
||||
description=DESCRIPTION,
|
||||
usage=f"vllm {self.name} <component> [options]",
|
||||
)
|
||||
launch_subparsers = launch_parser.add_subparsers(
|
||||
required=True, dest="launch_component"
|
||||
)
|
||||
|
||||
for cmd_cls in LaunchSubcommandBase.__subclasses__():
|
||||
cmd_subparser = launch_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
description=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
|
||||
cmd_cls.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"{self.name} {cmd_cls.name}"
|
||||
)
|
||||
|
||||
return launch_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [LaunchSubcommand()]
|
||||
|
||||
|
||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.engine.launch import LaunchEngineClient
|
||||
|
||||
# 1. Socket binding
|
||||
listen_address, sock = setup_server(args)
|
||||
|
||||
# 2. Create LaunchEngineClient (no GPU)
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
model_config = engine_args.create_model_config()
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
engine_client = LaunchEngineClient.from_vllm_config(vllm_config)
|
||||
|
||||
# 3. Build app, initialize state, and start serving
|
||||
shutdown_task = await build_and_serve(engine_client, listen_address, sock, args)
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
|
||||
def main():
|
||||
import vllm.entrypoints.cli.benchmark.main
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.launch
|
||||
import vllm.entrypoints.cli.openai
|
||||
import vllm.entrypoints.cli.run_batch
|
||||
import vllm.entrypoints.cli.serve
|
||||
@@ -25,6 +26,7 @@ def main():
|
||||
CMD_MODULES = [
|
||||
vllm.entrypoints.cli.openai,
|
||||
vllm.entrypoints.cli.serve,
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.benchmark.main,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.run_batch,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
import multiprocessing
|
||||
@@ -194,7 +195,7 @@ def build_app(
|
||||
|
||||
register_sagemaker_api_router(app, supported_tasks)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
if any(task in supported_tasks for task in ("generate", "render")):
|
||||
from vllm.entrypoints.openai.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
)
|
||||
@@ -357,7 +358,7 @@ async def init_app_state(
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
if any(task in supported_tasks for task in ("generate", "render")):
|
||||
from vllm.entrypoints.openai.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
@@ -469,6 +470,53 @@ def setup_server(args):
|
||||
return listen_address, sock
|
||||
|
||||
|
||||
async def build_and_serve(
|
||||
engine_client: EngineClient,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app, initialize state, and start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
app = build_app(args, supported_tasks)
|
||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server."""
|
||||
|
||||
@@ -490,47 +538,13 @@ async def run_server_worker(
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
client_config=client_config,
|
||||
) as engine_client:
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
app = build_app(args, supported_tasks)
|
||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||
|
||||
logger.info(
|
||||
"Starting vLLM API server %d on %s",
|
||||
engine_client.vllm_config.parallel_config._api_process_rank,
|
||||
listen_address,
|
||||
shutdown_task = await build_and_serve(
|
||||
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
||||
)
|
||||
shutdown_task = await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
# NB: Await server shutdown only after the backend context is exited
|
||||
try:
|
||||
await shutdown_task
|
||||
|
||||
@@ -113,7 +113,7 @@ async def init_generate_state(
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
if any(task in supported_tasks for task in ("generate", "render"))
|
||||
else None
|
||||
)
|
||||
# Warm up chat template processing to avoid first-request latency
|
||||
@@ -129,7 +129,7 @@ async def init_generate_state(
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
if any(task in supported_tasks for task in ("generate", "render"))
|
||||
else None
|
||||
)
|
||||
state.anthropic_serving_messages = (
|
||||
|
||||
Reference in New Issue
Block a user