[Frontend] Move api_server.py out openai folder (#52131)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
@@ -990,10 +990,12 @@ steps:
|
||||
- vllm/entrypoints
|
||||
- tests/entrypoints/unit_tests
|
||||
- tests/entrypoints/weight_transfer
|
||||
- tests/entrypoints/launchers
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -v -s entrypoints/unit_tests
|
||||
- pytest -v -s entrypoints/weight_transfer
|
||||
- pytest -v -s entrypoints/launchers
|
||||
|
||||
- label: Entrypoints Integration (LLM) # TBD
|
||||
timeout_in_minutes: 180
|
||||
@@ -2320,9 +2322,9 @@ steps:
|
||||
- tests/benchmarks/test_serve_cli.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
||||
- tests/entrypoints/launchers/test_shutdown.py
|
||||
- tests/entrypoints/openai/test_return_token_ids.py
|
||||
- tests/entrypoints/openai/test_uds.py
|
||||
- tests/entrypoints/serve/instrumentator/test_uds.py
|
||||
- tests/v1/sample/test_logprobs_e2e.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
@@ -2331,9 +2333,9 @@ steps:
|
||||
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
- pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||
- pytest -v -s entrypoints/openai/test_uds.py
|
||||
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
|
||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||
|
||||
- label: Rust Frontend Serve Admin Coverage # TBD
|
||||
|
||||
@@ -11,9 +11,11 @@ steps:
|
||||
- vllm/entrypoints
|
||||
- tests/entrypoints/unit_tests
|
||||
- tests/entrypoints/weight_transfer
|
||||
- tests/entrypoints/launchers
|
||||
commands:
|
||||
- pytest -v -s entrypoints/unit_tests
|
||||
- pytest -v -s entrypoints/weight_transfer
|
||||
- pytest -v -s entrypoints/launchers
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Entrypoints Unit"
|
||||
|
||||
@@ -18,9 +18,9 @@ steps:
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
|
||||
# - tests/entrypoints/openai/completion/test_prompt_validation.py
|
||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
||||
- tests/entrypoints/launchers/test_shutdown.py
|
||||
- tests/entrypoints/openai/test_return_token_ids.py
|
||||
- tests/entrypoints/openai/test_uds.py
|
||||
- tests/entrypoints/serve/instrumentator/test_uds.py
|
||||
- tests/v1/sample/test_logprobs_e2e.py
|
||||
commands:
|
||||
- export VLLM_USE_RUST_FRONTEND=1
|
||||
@@ -30,11 +30,11 @@ steps:
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||
|
||||
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
||||
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
- pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
# test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and
|
||||
# finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.
|
||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||
- pytest -v -s entrypoints/openai/test_uds.py
|
||||
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
|
||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||
|
||||
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
|
||||
|
||||
@@ -117,7 +117,7 @@ Commands:
|
||||
- >
|
||||
bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh
|
||||
leader --ray_cluster_size=2; python3 -m
|
||||
vllm.entrypoints.openai.api_server --port 8080 --model
|
||||
vllm.entrypoints.launchers.api_server.entry --port 8080 --model
|
||||
meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8
|
||||
--pipeline-parallel-size 2
|
||||
```
|
||||
|
||||
@@ -62,20 +62,6 @@ vllm serve <model>
|
||||
|
||||
The code for the `vllm` CLI can be found in [vllm/entrypoints/cli/main.py](../../vllm/entrypoints/cli/main.py).
|
||||
|
||||
Sometimes you may see the API server entrypoint used directly instead of via the
|
||||
`vllm` CLI command. For example:
|
||||
|
||||
```bash
|
||||
python -m vllm.entrypoints.openai.api_server --model <model>
|
||||
```
|
||||
|
||||
!!! warning
|
||||
|
||||
`python -m vllm.entrypoints.openai.api_server` is deprecated
|
||||
and may become unsupported in a future release.
|
||||
|
||||
That code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py).
|
||||
|
||||
More details on the API server can be found in the [Online Serving](../serving/online_serving/README.md) document.
|
||||
|
||||
## V1 Process Architecture
|
||||
@@ -88,7 +74,7 @@ The API server process handles HTTP requests (e.g., the OpenAI-compatible API),
|
||||
|
||||
By default, there is **1 API server process**, but when data parallelism is used, the API server count automatically scales to match the data parallel size. This can also be manually configured with the `--api-server-count` flag. Each API server connects to **all** engine cores via ZMQ in a many-to-many topology, enabling any API server to route requests to any engine core. Each API server process uses multiple CPU threads for media loading (controlled by `VLLM_MEDIA_LOADING_THREAD_COUNT`, default 8).
|
||||
|
||||
The code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py) and [vllm/v1/utils.py](../../vllm/v1/utils.py).
|
||||
The code can be found in [vllm/entrypoints/launchers/api_server](../../vllm/entrypoints/launchers/api_server) and [vllm/v1/utils.py](../../vllm/v1/utils.py).
|
||||
|
||||
### Engine Core Process
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Endpoint Plugins
|
||||
|
||||
Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server without editing `vllm/entrypoints/openai/api_server.py`. Their scope is
|
||||
Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server. Their scope is
|
||||
the **HTTP surface only** registering routes and optionally per app state used by those routes. A plugin reaches the engine the same way an in-tree serving handler does, through the `EngineClient` it is handed at startup (e.g. `engine_client.collective_rpc(...)`). No new engine access path is introduced.
|
||||
|
||||
!!! warning "Security"
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
NOTE: This API server is used only for demonstrating usage of AsyncEngine
|
||||
and simple performance benchmarks. It is not intended for production use.
|
||||
For production use, we recommend using our OpenAI compatible server.
|
||||
We are also not going to accept PRs modifying this file, please
|
||||
change `vllm/entrypoints/openai/api_server.py` instead.
|
||||
We are also not going to accept PRs modifying this file.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -21,7 +20,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
import vllm.envs as envs
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
from vllm.entrypoints.launcher import serve_http
|
||||
from vllm.entrypoints.launchers.launcher import serve_http
|
||||
from vllm.entrypoints.serve.utils.api_utils import with_cancellation
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from tests.entrypoints.unit_tests._api_server_spawn_workers import (
|
||||
from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
+1
-1
@@ -147,7 +147,7 @@ async def test_shutdown_on_engine_failure(tmp_path: Path):
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.openai.api_server",
|
||||
"vllm.entrypoints.launchers.api_server.entry",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--dtype",
|
||||
+1
-1
@@ -7,7 +7,7 @@ from ssl import SSLContext
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
|
||||
from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher
|
||||
|
||||
|
||||
class MockSSLContext(SSLContext):
|
||||
@@ -14,7 +14,7 @@ import pytest
|
||||
from fastapi import HTTPException
|
||||
from prometheus_client import CollectorRegistry
|
||||
|
||||
from vllm.entrypoints.openai.api_server import build_app
|
||||
from vllm.entrypoints.launchers.api_server.entry import build_app
|
||||
from vllm.exceptions import (
|
||||
VLLMNotFoundError,
|
||||
VLLMServerError,
|
||||
|
||||
+1
-2
@@ -6,11 +6,10 @@ from tempfile import TemporaryDirectory
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm import envs
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import time
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
@@ -7,7 +7,7 @@ Script to test add_lora, remove_lora, pin_lora, list_loras functions.
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
@@ -17,14 +17,14 @@ import pytest
|
||||
from fastapi import FastAPI
|
||||
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
|
||||
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
_attach_endpoint_plugins,
|
||||
_init_endpoint_plugins_state,
|
||||
build_app,
|
||||
)
|
||||
from vllm.entrypoints.launchers.app import build_app
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
from vllm.plugins import load_endpoint_plugins
|
||||
from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
|
||||
from vllm.plugins.endpoint_plugins.interface import (
|
||||
EndpointPlugin,
|
||||
attach_endpoint_plugins,
|
||||
init_endpoint_plugins_state,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@@ -157,7 +157,7 @@ def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch)
|
||||
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
|
||||
|
||||
app = FastAPI()
|
||||
_attach_endpoint_plugins(app, ("generate",))
|
||||
attach_endpoint_plugins(app, ("generate",))
|
||||
|
||||
assert app.state.endpoint_plugins == []
|
||||
|
||||
@@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
|
||||
state = State()
|
||||
await _init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
|
||||
await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
|
||||
|
||||
assert not hasattr(state, "dummy_engine_client")
|
||||
|
||||
@@ -196,7 +196,7 @@ async def test_render_server_attaches_endpoint_plugins_with_no_engine_client(
|
||||
for route in app.routes
|
||||
)
|
||||
|
||||
await _init_endpoint_plugins_state(None, app.state, args)
|
||||
await init_endpoint_plugins_state(None, app.state, args)
|
||||
|
||||
assert app.state.dummy_engine_client is None
|
||||
|
||||
@@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
|
||||
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
|
||||
await _init_endpoint_plugins_state(fake_engine_client, app.state, args)
|
||||
await init_endpoint_plugins_state(fake_engine_client, app.state, args)
|
||||
|
||||
assert app.state.dummy_engine_client is fake_engine_client
|
||||
|
||||
|
||||
@@ -305,7 +305,7 @@ async def run_vllm_async(
|
||||
disable_detokenize: bool = False,
|
||||
warmup_requests: list[SampleRequest] | None = None,
|
||||
) -> float:
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,18 +3,11 @@
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
import signal
|
||||
|
||||
import uvloop
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.api_server import (
|
||||
build_and_serve_renderer,
|
||||
setup_server,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
|
||||
from vllm.entrypoints.openai.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
@@ -120,38 +113,3 @@ class LaunchSubcommand(CLISubcommand):
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [LaunchSubcommand()]
|
||||
|
||||
|
||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
||||
# its own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
# 1. Socket binding
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
|
||||
# 2. Build and serve the API server
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
model_config = engine_args.create_model_config()
|
||||
|
||||
# Render servers preprocess data only — no inference, no quantized kernels.
|
||||
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
||||
model_config.quantization = None
|
||||
|
||||
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
||||
# cache space warning from CpuPlatform.check_and_update_config.
|
||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
shutdown_task = await build_and_serve_renderer(
|
||||
vllm_config, listen_address, sock, args
|
||||
)
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
@@ -10,11 +10,9 @@ import uvloop
|
||||
import vllm
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.api_server import run_server, setup_server
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
run_dp_supervisor,
|
||||
)
|
||||
from vllm.entrypoints.openai.dp_supervisor import run_dp_supervisor
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import warnings
|
||||
from argparse import Namespace
|
||||
from typing import cast
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
|
||||
|
||||
|
||||
async def init_app_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||
) -> None:
|
||||
vllm_config = engine_client.vllm_config
|
||||
|
||||
if args.tool_call_parser is not None:
|
||||
from vllm.parser.metrics import init_parser_metrics
|
||||
|
||||
init_parser_metrics(
|
||||
model_name=cast(str, vllm_config.model_config.served_model_name)
|
||||
)
|
||||
|
||||
if supported_tasks is None:
|
||||
warnings.warn(
|
||||
"The 'supported_tasks' parameter was not provided to "
|
||||
"init_app_state and will be required in a future version. "
|
||||
"Please pass 'supported_tasks' explicitly.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
supported_tasks = FALLBACK_SUPPORTED_TASKS
|
||||
|
||||
if args.served_model_name is not None:
|
||||
served_model_names = args.served_model_name
|
||||
else:
|
||||
served_model_names = [args.model]
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
base_model_paths = [
|
||||
BaseModelPath(name=name, model_path=args.model) for name in served_model_names
|
||||
]
|
||||
|
||||
state.engine_client = engine_client
|
||||
state.log_stats = not args.disable_log_stats
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (
|
||||
vllm_config.lora_config.default_mm_loras
|
||||
if vllm_config.lora_config is not None
|
||||
else {}
|
||||
)
|
||||
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
|
||||
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine_client=engine_client,
|
||||
base_model_paths=base_model_paths,
|
||||
lora_modules=lora_modules,
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||
|
||||
init_scale_out_state(state, args, engine_client, request_logger)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||
|
||||
init_speech_to_text_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_state
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
await init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
state.server_load_metrics = 0
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
import multiprocessing.forkserver as forkserver
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.system_utils import decorate_logs
|
||||
|
||||
from ..app import build_app
|
||||
from ..launcher import serve_http, setup_server
|
||||
from ..utils.server_utils import get_uvicorn_log_config
|
||||
from .app_state import init_app_state
|
||||
|
||||
prometheus_multiproc_dir: tempfile.TemporaryDirectory
|
||||
|
||||
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
|
||||
logger = init_logger("vllm.entrypoints.launchers.api_server.entry")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client(
|
||||
args: Namespace,
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
|
||||
# The executor is expected to be mp.
|
||||
# Pre-import heavy modules in the forkserver process
|
||||
logger.debug("Setup forkserver with pre-imports")
|
||||
multiprocessing.set_start_method("forkserver")
|
||||
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
|
||||
forkserver.ensure_running()
|
||||
logger.debug("Forkserver setup complete!")
|
||||
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
if client_config:
|
||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||
|
||||
async with build_async_engine_client_from_engine_args(
|
||||
engine_args,
|
||||
usage_context=usage_context,
|
||||
client_config=client_config,
|
||||
) as engine:
|
||||
yield engine
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client_from_engine_args(
|
||||
engine_args: AsyncEngineArgs,
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""
|
||||
Create EngineClient, either:
|
||||
- in-process using the AsyncLLMEngine Directly
|
||||
- multiprocess using AsyncLLMEngine RPC
|
||||
|
||||
Returns the Client or None if the creation failed.
|
||||
"""
|
||||
|
||||
# Create the EngineConfig (determines if we can use V1).
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
async_llm: AsyncLLM | None = None
|
||||
|
||||
# Don't mutate the input client_config
|
||||
client_config = dict(client_config) if client_config else {}
|
||||
client_count = client_config.pop("client_count", 1)
|
||||
client_index = client_config.pop("client_index", 0)
|
||||
|
||||
try:
|
||||
async_llm = AsyncLLM.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
enable_log_requests=engine_args.enable_log_requests,
|
||||
aggregate_engine_logging=engine_args.aggregate_engine_logging,
|
||||
disable_log_stats=engine_args.disable_log_stats,
|
||||
client_addresses=client_config,
|
||||
client_count=client_count,
|
||||
client_index=client_index,
|
||||
)
|
||||
|
||||
# Don't keep the dummy data in memory
|
||||
assert async_llm is not None
|
||||
await async_llm.reset_mm_cache()
|
||||
|
||||
yield async_llm
|
||||
finally:
|
||||
if async_llm:
|
||||
async_llm.shutdown(timeout=vllm_config.shutdown_timeout)
|
||||
|
||||
|
||||
async def build_and_serve(
|
||||
engine_client: EngineClient,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app, initialize state, and start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
model_config = engine_client.model_config
|
||||
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
app = build_app(args, supported_tasks, model_config)
|
||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server."""
|
||||
|
||||
decorate_logs("APIServer", skip_if_decorated=True)
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
||||
# own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
|
||||
|
||||
async def run_server_worker(
|
||||
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
||||
) -> None:
|
||||
"""Run a single API server worker."""
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
client_config=client_config,
|
||||
) as engine_client:
|
||||
shutdown_task = await build_and_serve(
|
||||
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
||||
)
|
||||
# NB: Await server shutdown only after the backend context is exited
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def main():
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.openai.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
# NOTE(simon):
|
||||
# This section should be in sync with vllm/entrypoints/cli/main.py for CLI
|
||||
# entrypoints.
|
||||
cli_env_setup()
|
||||
parser = FlexibleArgumentParser(
|
||||
description="vLLM OpenAI-Compatible RESTful API server."
|
||||
)
|
||||
parser = make_arg_parser(parser)
|
||||
args = parser.parse_args()
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
uvloop.run(run_server(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import warnings
|
||||
from argparse import Namespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
from .api_server.routers import register_api_routers
|
||||
from .utils.server_utils import lifespan
|
||||
|
||||
|
||||
def build_app(
|
||||
args: Namespace,
|
||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> FastAPI:
|
||||
if supported_tasks is None:
|
||||
warnings.warn(
|
||||
"The 'supported_tasks' parameter was not provided to "
|
||||
"build_app and will be required in a future version. "
|
||||
"Defaulting to ('generate',).",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
supported_tasks = FALLBACK_SUPPORTED_TASKS
|
||||
|
||||
if args.disable_fastapi_docs:
|
||||
app = FastAPI(
|
||||
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
||||
)
|
||||
elif args.enable_offline_docs:
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
else:
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.state.args = args
|
||||
app.root_path = args.root_path
|
||||
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugins are attached last so their routes are registered after all core
|
||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
app = sagemaker_standards_bootstrap(app)
|
||||
return app
|
||||
@@ -14,13 +14,23 @@ from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.constants import (
|
||||
from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
log_non_default_args,
|
||||
log_version_and_model,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.network_utils import find_process_using_port, is_valid_ipv6_address
|
||||
from vllm.utils.system_utils import set_ulimit
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
from .utils.constants import (
|
||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import find_process_using_port
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -190,3 +200,83 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
|
||||
engine_errored = engine.errored and not engine.is_running
|
||||
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
|
||||
server.should_exit = True
|
||||
|
||||
|
||||
def create_server_socket(
|
||||
addr: tuple[str, int],
|
||||
*,
|
||||
reuse_port: bool,
|
||||
) -> socket.socket:
|
||||
family = socket.AF_INET
|
||||
if is_valid_ipv6_address(addr[0]):
|
||||
family = socket.AF_INET6
|
||||
|
||||
sock = socket.socket(family=family, type=socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
if reuse_port:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
||||
sock.bind(addr)
|
||||
|
||||
return sock
|
||||
|
||||
|
||||
def create_server_unix_socket(path: str) -> socket.socket:
|
||||
sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
|
||||
sock.bind(path)
|
||||
return sock
|
||||
|
||||
|
||||
def validate_api_server_args(args):
|
||||
valid_tool_parses = ToolParserManager.list_registered()
|
||||
if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
|
||||
raise KeyError(
|
||||
f"invalid tool call parser: {args.tool_call_parser} "
|
||||
f"(chose from {{ {','.join(valid_tool_parses)} }})"
|
||||
)
|
||||
|
||||
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
||||
if (
|
||||
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
||||
) and reasoning_parser not in valid_reasoning_parsers:
|
||||
raise KeyError(
|
||||
f"invalid reasoning parser: {reasoning_parser} "
|
||||
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
||||
)
|
||||
|
||||
|
||||
@instrument(span_name="API server setup")
|
||||
def setup_server(args, *, reuse_port: bool):
|
||||
"""Validate API server args and create the server socket."""
|
||||
|
||||
log_version_and_model(logger, VLLM_VERSION, args.model)
|
||||
log_non_default_args(args)
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
validate_api_server_args(args)
|
||||
|
||||
# workaround to make sure that we bind the port before the engine is set up.
|
||||
# This avoids race conditions with ray.
|
||||
# see https://github.com/vllm-project/vllm/issues/8204
|
||||
if args.uds:
|
||||
sock = create_server_unix_socket(args.uds)
|
||||
else:
|
||||
sock_addr = (args.host or "", args.port)
|
||||
sock = create_server_socket(sock_addr, reuse_port=reuse_port)
|
||||
|
||||
# workaround to avoid footguns where uvicorn drops requests with too
|
||||
# many concurrent requests active
|
||||
set_ulimit()
|
||||
|
||||
if args.uds:
|
||||
listen_address = f"unix:{args.uds}"
|
||||
else:
|
||||
addr, port = sock_addr
|
||||
is_ssl = args.ssl_keyfile and args.ssl_certfile
|
||||
host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
|
||||
listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
|
||||
return listen_address, sock
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
|
||||
async def init_render_app_state(
|
||||
vllm_config: VllmConfig,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
"""Initialise FastAPI app state for a CPU-only render server.
|
||||
|
||||
Unlike :func:`init_app_state` this function does not require an
|
||||
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
||||
preprocessing pipeline (renderer, input_processor)
|
||||
directly from the :class:`~vllm.config.VllmConfig`.
|
||||
"""
|
||||
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
model_config=vllm_config.model_config,
|
||||
base_model_paths=[
|
||||
BaseModelPath(name=name, model_path=args.model)
|
||||
for name in served_model_names
|
||||
],
|
||||
)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
state.log_stats = False
|
||||
state.engine_client = None
|
||||
state.args = args
|
||||
state.enable_server_load_tracking = False
|
||||
state.server_load_metrics = 0
|
||||
|
||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||
await init_endpoint_plugins_state(None, state, args)
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import socket
|
||||
from argparse import Namespace
|
||||
|
||||
from vllm import AsyncEngineArgs, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..app import build_app
|
||||
from ..launcher import serve_http, setup_server
|
||||
from ..utils.server_utils import get_uvicorn_log_config
|
||||
from .app_state import init_render_app_state
|
||||
|
||||
logger = init_logger("vllm.entrypoints.launchers.render.entry")
|
||||
|
||||
|
||||
async def build_and_serve_renderer(
|
||||
vllm_config: VllmConfig,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
||||
start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
app = build_app(args, ("render",))
|
||||
await init_render_app_state(vllm_config, app.state, args)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
||||
# its own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
# 1. Socket binding
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
|
||||
# 2. Build and serve the API server
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
model_config = engine_args.create_model_config()
|
||||
|
||||
# Render servers preprocess data only — no inference, no quantized kernels.
|
||||
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
||||
model_config.quantization = None
|
||||
|
||||
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
||||
# cache space warning from CpuPlatform.check_and_update_config.
|
||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
shutdown_task = await build_and_serve_renderer(
|
||||
vllm_config, listen_address, sock, args
|
||||
)
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.openai.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
cli_env_setup()
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Starts a GPU-less rendering server "
|
||||
"for preprocessing and postprocessing only"
|
||||
)
|
||||
parser = make_arg_parser(parser)
|
||||
args = parser.parse_args()
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
-5
@@ -1,12 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared constants for vLLM entrypoints.
|
||||
"""
|
||||
|
||||
# HTTP header limits for h11 parser
|
||||
# These constants help mitigate header abuse attacks
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
|
||||
H11_MAX_HEADER_COUNT_DEFAULT = 256
|
||||
|
||||
MCP_PREFIX = "mcp_"
|
||||
+1
-1
@@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
@@ -1,670 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
import multiprocessing.forkserver as forkserver
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import tempfile
|
||||
import warnings
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, cast
|
||||
|
||||
import uvloop
|
||||
from fastapi import FastAPI
|
||||
from starlette.datastructures import State
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launcher import serve_http
|
||||
from vllm.entrypoints.launchers.api_server.app_state import init_app_state
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_and_serve,
|
||||
build_async_engine_client,
|
||||
build_async_engine_client_from_engine_args,
|
||||
run_server,
|
||||
run_server_worker,
|
||||
)
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
cli_env_setup,
|
||||
log_non_default_args,
|
||||
log_version_and_model,
|
||||
process_lora_modules,
|
||||
from vllm.entrypoints.launchers.app import build_app
|
||||
from vllm.entrypoints.launchers.launcher import (
|
||||
create_server_socket,
|
||||
create_server_unix_socket,
|
||||
setup_server,
|
||||
validate_api_server_args,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
get_uvicorn_log_config,
|
||||
lifespan,
|
||||
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
|
||||
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
|
||||
|
||||
warnings.warn(
|
||||
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
|
||||
"unsupported in a future version. Use the corresponding function from "
|
||||
"`vllm.entrypoints.launchers` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=1,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tracing import instrument
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.network_utils import is_valid_ipv6_address
|
||||
from vllm.utils.system_utils import decorate_logs, set_ulimit
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
prometheus_multiproc_dir: tempfile.TemporaryDirectory
|
||||
|
||||
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
|
||||
logger = init_logger("vllm.entrypoints.openai.api_server")
|
||||
|
||||
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
|
||||
|
||||
|
||||
def _attach_endpoint_plugins(
|
||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||
) -> None:
|
||||
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
|
||||
|
||||
Attached last after all core routers. This is so endpoint plugin routes can
|
||||
shadow core routes with the same path (see `EndpointPlugin.attach_router`
|
||||
docstring). No-ops when no plugins are discovered/allowlisted.
|
||||
"""
|
||||
from vllm.plugins import load_endpoint_plugins
|
||||
|
||||
endpoint_plugins = load_endpoint_plugins(supported_tasks)
|
||||
for plugin in endpoint_plugins:
|
||||
plugin.attach_router(app)
|
||||
app.state.endpoint_plugins = endpoint_plugins
|
||||
|
||||
|
||||
async def _init_endpoint_plugins_state(
|
||||
engine_client: EngineClient | None, state: State, args: Namespace
|
||||
) -> None:
|
||||
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
|
||||
|
||||
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
|
||||
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
|
||||
build their own bare `State` without going through `build_app`. As a result
|
||||
`endpoint_plugins` may be absent and are treated that the same as "none attached".
|
||||
|
||||
`engine_client` is `None` for the CPU only render server which has no
|
||||
engine (see `init_render_app_state`). Plugins must handle a `None`
|
||||
`engine_client` themselves (see `EndpointPlugin.init_state`).
|
||||
"""
|
||||
for plugin in getattr(state, "endpoint_plugins", []):
|
||||
await plugin.init_state(engine_client, state, args)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client(
|
||||
args: Namespace,
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
|
||||
# The executor is expected to be mp.
|
||||
# Pre-import heavy modules in the forkserver process
|
||||
logger.debug("Setup forkserver with pre-imports")
|
||||
multiprocessing.set_start_method("forkserver")
|
||||
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
|
||||
forkserver.ensure_running()
|
||||
logger.debug("Forkserver setup complete!")
|
||||
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
if client_config:
|
||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||
|
||||
async with build_async_engine_client_from_engine_args(
|
||||
engine_args,
|
||||
usage_context=usage_context,
|
||||
client_config=client_config,
|
||||
) as engine:
|
||||
yield engine
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client_from_engine_args(
|
||||
engine_args: AsyncEngineArgs,
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""
|
||||
Create EngineClient, either:
|
||||
- in-process using the AsyncLLMEngine Directly
|
||||
- multiprocess using AsyncLLMEngine RPC
|
||||
|
||||
Returns the Client or None if the creation failed.
|
||||
"""
|
||||
|
||||
# Create the EngineConfig (determines if we can use V1).
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
async_llm: AsyncLLM | None = None
|
||||
|
||||
# Don't mutate the input client_config
|
||||
client_config = dict(client_config) if client_config else {}
|
||||
client_count = client_config.pop("client_count", 1)
|
||||
client_index = client_config.pop("client_index", 0)
|
||||
|
||||
try:
|
||||
async_llm = AsyncLLM.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=usage_context,
|
||||
enable_log_requests=engine_args.enable_log_requests,
|
||||
aggregate_engine_logging=engine_args.aggregate_engine_logging,
|
||||
disable_log_stats=engine_args.disable_log_stats,
|
||||
client_addresses=client_config,
|
||||
client_count=client_count,
|
||||
client_index=client_index,
|
||||
)
|
||||
|
||||
# Don't keep the dummy data in memory
|
||||
assert async_llm is not None
|
||||
await async_llm.reset_mm_cache()
|
||||
|
||||
yield async_llm
|
||||
finally:
|
||||
if async_llm:
|
||||
async_llm.shutdown(timeout=vllm_config.shutdown_timeout)
|
||||
|
||||
|
||||
def build_app(
|
||||
args: Namespace,
|
||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> FastAPI:
|
||||
if supported_tasks is None:
|
||||
warnings.warn(
|
||||
"The 'supported_tasks' parameter was not provided to "
|
||||
"build_app and will be required in a future version. "
|
||||
"Defaulting to ('generate',).",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
supported_tasks = _FALLBACK_SUPPORTED_TASKS
|
||||
|
||||
if args.disable_fastapi_docs:
|
||||
app = FastAPI(
|
||||
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
||||
)
|
||||
elif args.enable_offline_docs:
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
else:
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.state.args = args
|
||||
app.root_path = args.root_path
|
||||
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugins are attached last so their routes are registered after all core
|
||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
_attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
app = sagemaker_standards_bootstrap(app)
|
||||
return app
|
||||
|
||||
|
||||
async def init_app_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||
) -> None:
|
||||
vllm_config = engine_client.vllm_config
|
||||
|
||||
if args.tool_call_parser is not None:
|
||||
from vllm.parser.metrics import init_parser_metrics
|
||||
|
||||
init_parser_metrics(
|
||||
model_name=cast(str, vllm_config.model_config.served_model_name)
|
||||
)
|
||||
|
||||
if supported_tasks is None:
|
||||
warnings.warn(
|
||||
"The 'supported_tasks' parameter was not provided to "
|
||||
"init_app_state and will be required in a future version. "
|
||||
"Please pass 'supported_tasks' explicitly.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
supported_tasks = _FALLBACK_SUPPORTED_TASKS
|
||||
|
||||
if args.served_model_name is not None:
|
||||
served_model_names = args.served_model_name
|
||||
else:
|
||||
served_model_names = [args.model]
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
base_model_paths = [
|
||||
BaseModelPath(name=name, model_path=args.model) for name in served_model_names
|
||||
]
|
||||
|
||||
state.engine_client = engine_client
|
||||
state.log_stats = not args.disable_log_stats
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (
|
||||
vllm_config.lora_config.default_mm_loras
|
||||
if vllm_config.lora_config is not None
|
||||
else {}
|
||||
)
|
||||
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
|
||||
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine_client=engine_client,
|
||||
base_model_paths=base_model_paths,
|
||||
lora_modules=lora_modules,
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||
|
||||
init_scale_out_state(state, args, engine_client, request_logger)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||
|
||||
init_speech_to_text_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_state
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
await _init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
state.server_load_metrics = 0
|
||||
|
||||
|
||||
async def init_render_app_state(
|
||||
vllm_config: VllmConfig,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
"""Initialise FastAPI app state for a CPU-only render server.
|
||||
|
||||
Unlike :func:`init_app_state` this function does not require an
|
||||
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
||||
preprocessing pipeline (renderer, input_processor)
|
||||
directly from the :class:`~vllm.config.VllmConfig`.
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
model_config=vllm_config.model_config,
|
||||
base_model_paths=[
|
||||
BaseModelPath(name=name, model_path=args.model)
|
||||
for name in served_model_names
|
||||
],
|
||||
)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
state.log_stats = False
|
||||
state.engine_client = None
|
||||
state.args = args
|
||||
state.enable_server_load_tracking = False
|
||||
state.server_load_metrics = 0
|
||||
|
||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||
await _init_endpoint_plugins_state(None, state, args)
|
||||
|
||||
|
||||
def create_server_socket(
|
||||
addr: tuple[str, int],
|
||||
*,
|
||||
reuse_port: bool,
|
||||
) -> socket.socket:
|
||||
family = socket.AF_INET
|
||||
if is_valid_ipv6_address(addr[0]):
|
||||
family = socket.AF_INET6
|
||||
|
||||
sock = socket.socket(family=family, type=socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
if reuse_port:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
||||
sock.bind(addr)
|
||||
|
||||
return sock
|
||||
|
||||
|
||||
def create_server_unix_socket(path: str) -> socket.socket:
|
||||
sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
|
||||
sock.bind(path)
|
||||
return sock
|
||||
|
||||
|
||||
def validate_api_server_args(args):
|
||||
valid_tool_parses = ToolParserManager.list_registered()
|
||||
if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
|
||||
raise KeyError(
|
||||
f"invalid tool call parser: {args.tool_call_parser} "
|
||||
f"(chose from {{ {','.join(valid_tool_parses)} }})"
|
||||
)
|
||||
|
||||
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
||||
if (
|
||||
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
||||
) and reasoning_parser not in valid_reasoning_parsers:
|
||||
raise KeyError(
|
||||
f"invalid reasoning parser: {reasoning_parser} "
|
||||
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
||||
)
|
||||
|
||||
|
||||
@instrument(span_name="API server setup")
|
||||
def setup_server(args, *, reuse_port: bool):
|
||||
"""Validate API server args and create the server socket."""
|
||||
|
||||
log_version_and_model(logger, VLLM_VERSION, args.model)
|
||||
log_non_default_args(args)
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
validate_api_server_args(args)
|
||||
|
||||
# workaround to make sure that we bind the port before the engine is set up.
|
||||
# This avoids race conditions with ray.
|
||||
# see https://github.com/vllm-project/vllm/issues/8204
|
||||
if args.uds:
|
||||
sock = create_server_unix_socket(args.uds)
|
||||
else:
|
||||
sock_addr = (args.host or "", args.port)
|
||||
sock = create_server_socket(sock_addr, reuse_port=reuse_port)
|
||||
|
||||
# workaround to avoid footguns where uvicorn drops requests with too
|
||||
# many concurrent requests active
|
||||
set_ulimit()
|
||||
|
||||
if args.uds:
|
||||
listen_address = f"unix:{args.uds}"
|
||||
else:
|
||||
addr, port = sock_addr
|
||||
is_ssl = args.ssl_keyfile and args.ssl_certfile
|
||||
host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
|
||||
listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
|
||||
return listen_address, sock
|
||||
|
||||
|
||||
async def build_and_serve(
|
||||
engine_client: EngineClient,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app, initialize state, and start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
model_config = engine_client.model_config
|
||||
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
app = build_app(args, supported_tasks, model_config)
|
||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def build_and_serve_renderer(
|
||||
vllm_config: VllmConfig,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
||||
start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
app = build_app(args, ("render",))
|
||||
await init_render_app_state(vllm_config, app.state, args)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server."""
|
||||
|
||||
decorate_logs("APIServer", skip_if_decorated=True)
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
||||
# own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
|
||||
|
||||
async def run_server_worker(
|
||||
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
||||
) -> None:
|
||||
"""Run a single API server worker."""
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
|
||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
client_config=client_config,
|
||||
) as engine_client:
|
||||
shutdown_task = await build_and_serve(
|
||||
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
||||
)
|
||||
# NB: Await server shutdown only after the backend context is exited
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
__all__ = [
|
||||
"build_async_engine_client",
|
||||
"build_async_engine_client_from_engine_args",
|
||||
"build_app",
|
||||
"init_app_state",
|
||||
"init_render_app_state",
|
||||
"create_server_socket",
|
||||
"create_server_unix_socket",
|
||||
"validate_api_server_args",
|
||||
"setup_server",
|
||||
"build_and_serve",
|
||||
"build_and_serve_renderer",
|
||||
"run_server",
|
||||
"run_server_worker",
|
||||
"register_api_routers",
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
# NOTE(simon):
|
||||
# This section should be in sync with vllm/entrypoints/cli/main.py for CLI
|
||||
# entrypoints.
|
||||
cli_env_setup()
|
||||
parser = FlexibleArgumentParser(
|
||||
description="vLLM OpenAI-Compatible RESTful API server."
|
||||
warnings.warn(
|
||||
"The `python -m vllm.entrypoints.openai.api_server` command is deprecated "
|
||||
"and may be removed in a future release. Please use `vllm server` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=1,
|
||||
)
|
||||
parser = make_arg_parser(parser)
|
||||
args = parser.parse_args()
|
||||
validate_parsed_serve_args(args)
|
||||
from vllm.entrypoints.launchers.api_server.entry import main
|
||||
|
||||
uvloop.run(run_server(args))
|
||||
main()
|
||||
|
||||
@@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
validate_chat_template,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.entrypoints.serve.utils.constants import (
|
||||
from vllm.entrypoints.launchers.utils.constants import (
|
||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
@@ -444,6 +444,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
|
||||
def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
parser_for_docs = FlexibleArgumentParser(
|
||||
prog="-m vllm.entrypoints.openai.api_server"
|
||||
prog="-m vllm.entrypoints.launchers.api_server.entry"
|
||||
)
|
||||
return make_arg_parser(parser_for_docs)
|
||||
|
||||
@@ -23,7 +23,7 @@ import uvloop
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.launcher import NoSignalServer
|
||||
from vllm.entrypoints.launchers.launcher import NoSignalServer
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.system_utils import (
|
||||
decorate_logs,
|
||||
@@ -236,7 +236,7 @@ def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
|
||||
|
||||
|
||||
def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.openai.api_server import run_server
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server
|
||||
|
||||
uvloop.run(run_server(child_args))
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
construct_tool_dicts,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -58,6 +57,7 @@ _TOOL_NAME_TO_TYPE_MAP = {
|
||||
"python": "code_interpreter",
|
||||
"container": "container",
|
||||
}
|
||||
MCP_PREFIX = "mcp_"
|
||||
|
||||
|
||||
def _map_tool_name_to_tool_type(tool_name: str) -> str:
|
||||
|
||||
@@ -31,7 +31,7 @@ from vllm.config import config
|
||||
from vllm.connections import global_http_connection
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.api_server import init_app_state
|
||||
from vllm.entrypoints.launchers.api_server.entry import init_app_state
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
@@ -836,8 +836,7 @@ async def run_batch(
|
||||
error_msg=f"URL {request.url} was used. "
|
||||
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
|
||||
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
|
||||
" /rerank. See vllm/entrypoints/openai/api_server.py "
|
||||
"for supported score/rerank versions.",
|
||||
" /rerank.",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -848,7 +847,7 @@ async def run_batch(
|
||||
|
||||
|
||||
async def main(args: Namespace):
|
||||
from vllm.entrypoints.openai.api_server import build_async_engine_client
|
||||
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
validate_run_batch_args(args)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from fastapi import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.launchers.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.exceptions import VLLMError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -20,7 +20,7 @@ for the loading/gating rules and `docs/usage/security.md` for the security
|
||||
posture of exposing plugin defined routes.
|
||||
|
||||
The CPU only render server (see `build_and_serve_renderer` in
|
||||
`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin
|
||||
`vllm/entrypoints/launchers/render`) has no `EngineClient`. A plugin
|
||||
eligible for the `render` task (`required_tasks` is `None` or includes
|
||||
`"render"`) still gets `attach_router` called but `init_state` receives
|
||||
`engine_client=None`. Plugins that cannot function without an engine should
|
||||
@@ -34,8 +34,9 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
from fastapi import FastAPI
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
|
||||
@@ -85,3 +86,38 @@ class EndpointPlugin(Protocol):
|
||||
cannot function without an engine.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def attach_endpoint_plugins(
|
||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||
) -> None:
|
||||
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
|
||||
|
||||
Attached last after all core routers. This is so endpoint plugin routes can
|
||||
shadow core routes with the same path (see `EndpointPlugin.attach_router`
|
||||
docstring). No-ops when no plugins are discovered/allowlisted.
|
||||
"""
|
||||
from vllm.plugins import load_endpoint_plugins
|
||||
|
||||
endpoint_plugins = load_endpoint_plugins(supported_tasks)
|
||||
for plugin in endpoint_plugins:
|
||||
plugin.attach_router(app)
|
||||
app.state.endpoint_plugins = endpoint_plugins
|
||||
|
||||
|
||||
async def init_endpoint_plugins_state(
|
||||
engine_client: EngineClient | None, state: State, args: Namespace
|
||||
) -> None:
|
||||
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
|
||||
|
||||
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
|
||||
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
|
||||
build their own bare `State` without going through `build_app`. As a result
|
||||
`endpoint_plugins` may be absent and are treated that the same as "none attached".
|
||||
|
||||
`engine_client` is `None` for the CPU only render server which has no
|
||||
engine (see `init_render_app_state`). Plugins must handle a `None`
|
||||
`engine_client` themselves (see `EndpointPlugin.init_state`).
|
||||
"""
|
||||
for plugin in getattr(state, "endpoint_plugins", []):
|
||||
await plugin.init_state(engine_client, state, args)
|
||||
|
||||
@@ -41,3 +41,4 @@ FrontendTask = Literal["render"]
|
||||
FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)
|
||||
|
||||
SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]
|
||||
FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
|
||||
|
||||
+1
-1
@@ -503,7 +503,7 @@ def run_api_server_worker_proc(
|
||||
) -> None:
|
||||
"""Entrypoint for individual API server worker processes."""
|
||||
|
||||
from vllm.entrypoints.openai.api_server import run_server_worker
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server_worker
|
||||
|
||||
client_config = client_config or {}
|
||||
server_index = client_config.get("client_index", 0)
|
||||
|
||||
Reference in New Issue
Block a user