[Frontend] Add vllm launch command for GPU-less preprocessing serving (#34551)

Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
Hyunkyun Moon
2026-03-04 18:41:52 +00:00
committed by GitHub
parent 32224f568a
commit bc6be89d16
10 changed files with 776 additions and 65 deletions
+128
View File
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import uvloop
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.openai.api_server import (
build_and_serve,
setup_server,
)
from vllm.entrypoints.openai.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
DESCRIPTION = "Launch individual vLLM components."
class LaunchSubcommandBase(CLISubcommand):
"""The base class of subcommands for `vllm launch`."""
help: str
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser.
By default, adds the standard vLLM serving arguments.
Subclasses can override to add component-specific arguments.
"""
make_arg_parser(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
raise NotImplementedError
class RenderSubcommand(LaunchSubcommandBase):
"""The `render` subcommand for `vllm launch`."""
name = "render"
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
@staticmethod
def cmd(args: argparse.Namespace) -> None:
uvloop.run(run_launch_fastapi(args))
class LaunchSubcommand(CLISubcommand):
"""The `launch` subcommand for the vLLM CLI.
Uses nested sub-subcommands so each component can define its own
arguments independently (e.g. ``vllm launch render``).
"""
name = "launch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
if hasattr(args, "model_tag") and args.model_tag is not None:
args.model = args.model_tag
args.launch_command(args)
def validate(self, args: argparse.Namespace) -> None:
validate_parsed_serve_args(args)
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
launch_parser = subparsers.add_parser(
self.name,
help=DESCRIPTION,
description=DESCRIPTION,
usage=f"vllm {self.name} <component> [options]",
)
launch_subparsers = launch_parser.add_subparsers(
required=True, dest="launch_component"
)
for cmd_cls in LaunchSubcommandBase.__subclasses__():
cmd_subparser = launch_subparsers.add_parser(
cmd_cls.name,
help=cmd_cls.help,
description=cmd_cls.help,
usage=f"vllm {self.name} {cmd_cls.name} [options]",
)
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}"
)
return launch_parser
def cmd_init() -> list[CLISubcommand]:
return [LaunchSubcommand()]
async def run_launch_fastapi(args: argparse.Namespace) -> None:
"""Run the online serving layer with FastAPI (no GPU inference)."""
from vllm.config import VllmConfig
from vllm.v1.engine.launch import LaunchEngineClient
# 1. Socket binding
listen_address, sock = setup_server(args)
# 2. Create LaunchEngineClient (no GPU)
engine_args = AsyncEngineArgs.from_cli_args(args)
model_config = engine_args.create_model_config()
vllm_config = VllmConfig(model_config=model_config)
engine_client = LaunchEngineClient.from_vllm_config(vllm_config)
# 3. Build app, initialize state, and start serving
shutdown_task = await build_and_serve(engine_client, listen_address, sock, args)
try:
await shutdown_task
finally:
sock.close()
+2
View File
@@ -16,6 +16,7 @@ logger = init_logger(__name__)
def main():
import vllm.entrypoints.cli.benchmark.main
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.launch
import vllm.entrypoints.cli.openai
import vllm.entrypoints.cli.run_batch
import vllm.entrypoints.cli.serve
@@ -25,6 +26,7 @@ def main():
CMD_MODULES = [
vllm.entrypoints.cli.openai,
vllm.entrypoints.cli.serve,
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.benchmark.main,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.run_batch,
+52 -38
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import importlib
import inspect
import multiprocessing
@@ -194,7 +195,7 @@ def build_app(
register_sagemaker_api_router(app, supported_tasks)
if "generate" in supported_tasks:
if any(task in supported_tasks for task in ("generate", "render")):
from vllm.entrypoints.openai.generate.api_router import (
register_generate_api_routers,
)
@@ -357,7 +358,7 @@ async def init_app_state(
log_error_stack=args.log_error_stack,
)
if "generate" in supported_tasks:
if any(task in supported_tasks for task in ("generate", "render")):
from vllm.entrypoints.openai.generate.api_router import init_generate_state
await init_generate_state(
@@ -469,6 +470,53 @@ def setup_server(args):
return listen_address, sock
async def build_and_serve(
engine_client: EngineClient,
listen_address: str,
sock: socket.socket,
args: Namespace,
**uvicorn_kwargs,
) -> asyncio.Task:
"""Build FastAPI app, initialize state, and start serving.
Returns the shutdown task for the caller to await.
"""
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
uvicorn_kwargs["log_config"] = log_config
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
app = build_app(args, supported_tasks)
await init_app_state(engine_client, app.state, args, supported_tasks)
logger.info("Starting vLLM server on %s", listen_address)
return await serve_http(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
# NOTE: When the 'disable_uvicorn_access_log' value is True,
# no access log will be output.
access_log=not args.disable_uvicorn_access_log,
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
ssl_keyfile=args.ssl_keyfile,
ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs,
ssl_ciphers=args.ssl_ciphers,
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
h11_max_header_count=args.h11_max_header_count,
**uvicorn_kwargs,
)
async def run_server(args, **uvicorn_kwargs) -> None:
"""Run a single-worker API server."""
@@ -490,47 +538,13 @@ async def run_server_worker(
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
uvicorn_kwargs["log_config"] = log_config
async with build_async_engine_client(
args,
client_config=client_config,
) as engine_client:
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
app = build_app(args, supported_tasks)
await init_app_state(engine_client, app.state, args, supported_tasks)
logger.info(
"Starting vLLM API server %d on %s",
engine_client.vllm_config.parallel_config._api_process_rank,
listen_address,
shutdown_task = await build_and_serve(
engine_client, listen_address, sock, args, **uvicorn_kwargs
)
shutdown_task = await serve_http(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
# NOTE: When the 'disable_uvicorn_access_log' value is True,
# no access log will be output.
access_log=not args.disable_uvicorn_access_log,
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
ssl_keyfile=args.ssl_keyfile,
ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs,
ssl_ciphers=args.ssl_ciphers,
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
h11_max_header_count=args.h11_max_header_count,
**uvicorn_kwargs,
)
# NB: Await server shutdown only after the backend context is exited
try:
await shutdown_task
@@ -113,7 +113,7 @@ async def init_generate_state(
enable_log_deltas=args.enable_log_deltas,
log_error_stack=args.log_error_stack,
)
if "generate" in supported_tasks
if any(task in supported_tasks for task in ("generate", "render"))
else None
)
# Warm up chat template processing to avoid first-request latency
@@ -129,7 +129,7 @@ async def init_generate_state(
enable_force_include_usage=args.enable_force_include_usage,
log_error_stack=args.log_error_stack,
)
if "generate" in supported_tasks
if any(task in supported_tasks for task in ("generate", "render"))
else None
)
state.anthropic_serving_messages = (
+4 -1
View File
@@ -10,4 +10,7 @@ PoolingTask = Literal[
]
POOLING_TASKS: tuple[PoolingTask, ...] = get_args(PoolingTask)
SupportedTask = Literal[GenerationTask, PoolingTask]
FrontendTask = Literal["render"]
FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)
SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]
+201
View File
@@ -0,0 +1,201 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
LaunchEngineClient: A lightweight EngineClient for GPU-less online serving.
This implements the EngineClient protocol without AsyncLLM or EngineCore,
enabling preprocessing (tokenization, rendering) and postprocessing
(detokenization) without GPU inference.
"""
from collections.abc import AsyncGenerator, Iterable, Mapping
from typing import Any
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.inputs import ProcessorInputs, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.outputs import PoolingRequestOutput, RequestOutput
from vllm.plugins.io_processors import get_io_processor
from vllm.pooling_params import PoolingParams
from vllm.renderers import renderer_from_config
from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.v1.engine import EngineCoreRequest, PauseMode
from vllm.v1.engine.input_processor import InputProcessor
logger = init_logger(__name__)
class LaunchEngineClient(EngineClient):
"""GPU-less EngineClient that only supports preprocessing/postprocessing.
This is a Null Object at the EngineClient level, bypassing AsyncLLM
entirely. It initializes renderer, io_processor, and input_processor
for tokenization and rendering, but raises NotImplementedError for
any inference-related operations.
"""
def __init__(
self,
vllm_config: VllmConfig,
) -> None:
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.renderer = renderer = renderer_from_config(self.vllm_config)
self.io_processor = get_io_processor(
self.vllm_config,
self.renderer,
self.model_config.io_processor_plugin,
)
# Convert TokPrompt --> EngineCoreRequest.
self.input_processor = InputProcessor(self.vllm_config, renderer)
@classmethod
def from_vllm_config(
cls,
vllm_config: VllmConfig,
) -> "LaunchEngineClient":
"""Create a LaunchEngineClient from a VllmConfig without GPU."""
return cls(
vllm_config=vllm_config,
)
# -- Task support --
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
return ("render",)
# -- Inference (not supported) --
async def generate(
self,
prompt: EngineCoreRequest
| PromptType
| ProcessorInputs
| AsyncGenerator[StreamingInput, None],
sampling_params: SamplingParams,
request_id: str,
*,
prompt_text: str | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
data_parallel_rank: int | None = None,
reasoning_ended: bool | None = None,
) -> AsyncGenerator[RequestOutput, None]:
raise NotImplementedError(
"LaunchEngineClient does not support inference. "
"Use vllm serve for generation requests."
)
# yield is needed to make this an async generator
yield # type: ignore[misc] # pragma: no cover
# -- Request management (no-op) --
async def abort(
self, request_id: str | Iterable[str], internal: bool = False
) -> None:
pass
# -- Generation control (no-op) --
async def pause_generation(
self,
*,
mode: PauseMode = "abort",
wait_for_inflight_requests: bool | None = None,
clear_cache: bool = True,
) -> None:
pass
async def resume_generation(self) -> None:
pass
async def is_paused(self) -> bool:
return False
async def encode(
self,
prompt: PromptType | ProcessorInputs,
pooling_params: PoolingParams,
request_id: str,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
tokenization_kwargs: dict[str, Any] | None = None,
reasoning_ended: bool | None = None,
) -> AsyncGenerator[PoolingRequestOutput, None]:
raise NotImplementedError(
"LaunchEngineClient does not support inference. "
"Use vllm serve for encoding requests."
)
yield # type: ignore[misc] # pragma: no cover
# -- Observability (no-op / defaults) --
async def is_tracing_enabled(self) -> bool:
return False
async def do_log_stats(self) -> None:
pass
async def check_health(self) -> None:
pass
async def start_profile(self) -> None:
pass
async def stop_profile(self) -> None:
pass
# -- Cache management (no-op) --
async def reset_mm_cache(self) -> None:
pass
async def reset_prefix_cache(
self, reset_running_requests: bool = False, reset_connector: bool = False
) -> bool:
return True
async def reset_encoder_cache(self) -> None:
pass
# -- Power management (no-op) --
async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
pass
async def wake_up(self, tags: list[str] | None = None) -> None:
pass
async def is_sleeping(self) -> bool:
return False
# -- LoRA (not supported) --
async def add_lora(self, lora_request: LoRARequest) -> bool:
return False
# -- Status properties --
@property
def is_running(self) -> bool:
return True
@property
def is_stopped(self) -> bool:
return False
@property
def errored(self) -> bool:
return False
@property
def dead_error(self) -> BaseException:
return RuntimeError("LaunchEngineClient does not support inference")