[Frontend] Move api_server.py out openai folder (#52131)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -990,10 +990,12 @@ steps:
|
|||||||
- vllm/entrypoints
|
- vllm/entrypoints
|
||||||
- tests/entrypoints/unit_tests
|
- tests/entrypoints/unit_tests
|
||||||
- tests/entrypoints/weight_transfer
|
- tests/entrypoints/weight_transfer
|
||||||
|
- tests/entrypoints/launchers
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/unit_tests
|
- pytest -v -s entrypoints/unit_tests
|
||||||
- pytest -v -s entrypoints/weight_transfer
|
- pytest -v -s entrypoints/weight_transfer
|
||||||
|
- pytest -v -s entrypoints/launchers
|
||||||
|
|
||||||
- label: Entrypoints Integration (LLM) # TBD
|
- label: Entrypoints Integration (LLM) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -2320,9 +2322,9 @@ steps:
|
|||||||
- tests/benchmarks/test_serve_cli.py
|
- tests/benchmarks/test_serve_cli.py
|
||||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
- tests/entrypoints/launchers/test_shutdown.py
|
||||||
- tests/entrypoints/openai/test_return_token_ids.py
|
- tests/entrypoints/openai/test_return_token_ids.py
|
||||||
- tests/entrypoints/openai/test_uds.py
|
- tests/entrypoints/serve/instrumentator/test_uds.py
|
||||||
- tests/v1/sample/test_logprobs_e2e.py
|
- tests/v1/sample/test_logprobs_e2e.py
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
@@ -2331,9 +2333,9 @@ steps:
|
|||||||
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
||||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
|
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
|
||||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||||
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
- pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||||
- pytest -v -s entrypoints/openai/test_uds.py
|
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
|
||||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||||
|
|
||||||
- label: Rust Frontend Serve Admin Coverage # TBD
|
- label: Rust Frontend Serve Admin Coverage # TBD
|
||||||
|
|||||||
@@ -11,9 +11,11 @@ steps:
|
|||||||
- vllm/entrypoints
|
- vllm/entrypoints
|
||||||
- tests/entrypoints/unit_tests
|
- tests/entrypoints/unit_tests
|
||||||
- tests/entrypoints/weight_transfer
|
- tests/entrypoints/weight_transfer
|
||||||
|
- tests/entrypoints/launchers
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s entrypoints/unit_tests
|
- pytest -v -s entrypoints/unit_tests
|
||||||
- pytest -v -s entrypoints/weight_transfer
|
- pytest -v -s entrypoints/weight_transfer
|
||||||
|
- pytest -v -s entrypoints/launchers
|
||||||
mirror:
|
mirror:
|
||||||
amd:
|
amd:
|
||||||
label: ":amd: (MI355) Entrypoints Unit"
|
label: ":amd: (MI355) Entrypoints Unit"
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ steps:
|
|||||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||||
|
|
||||||
# - tests/entrypoints/openai/completion/test_prompt_validation.py
|
# - tests/entrypoints/openai/completion/test_prompt_validation.py
|
||||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
- tests/entrypoints/launchers/test_shutdown.py
|
||||||
- tests/entrypoints/openai/test_return_token_ids.py
|
- tests/entrypoints/openai/test_return_token_ids.py
|
||||||
- tests/entrypoints/openai/test_uds.py
|
- tests/entrypoints/serve/instrumentator/test_uds.py
|
||||||
- tests/v1/sample/test_logprobs_e2e.py
|
- tests/v1/sample/test_logprobs_e2e.py
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_USE_RUST_FRONTEND=1
|
- export VLLM_USE_RUST_FRONTEND=1
|
||||||
@@ -30,11 +30,11 @@ steps:
|
|||||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||||
|
|
||||||
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
||||||
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
- pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||||
# test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and
|
# test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and
|
||||||
# finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.
|
# finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.
|
||||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||||
- pytest -v -s entrypoints/openai/test_uds.py
|
- pytest -v -s entrypoints/serve/instrumentator/test_uds.py
|
||||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||||
|
|
||||||
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
|
- label: ":nvidia: (H200) Rust Frontend Serve/Admin Coverage"
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ Commands:
|
|||||||
- >
|
- >
|
||||||
bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh
|
bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh
|
||||||
leader --ray_cluster_size=2; python3 -m
|
leader --ray_cluster_size=2; python3 -m
|
||||||
vllm.entrypoints.openai.api_server --port 8080 --model
|
vllm.entrypoints.launchers.api_server.entry --port 8080 --model
|
||||||
meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8
|
meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8
|
||||||
--pipeline-parallel-size 2
|
--pipeline-parallel-size 2
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -62,20 +62,6 @@ vllm serve <model>
|
|||||||
|
|
||||||
The code for the `vllm` CLI can be found in [vllm/entrypoints/cli/main.py](../../vllm/entrypoints/cli/main.py).
|
The code for the `vllm` CLI can be found in [vllm/entrypoints/cli/main.py](../../vllm/entrypoints/cli/main.py).
|
||||||
|
|
||||||
Sometimes you may see the API server entrypoint used directly instead of via the
|
|
||||||
`vllm` CLI command. For example:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m vllm.entrypoints.openai.api_server --model <model>
|
|
||||||
```
|
|
||||||
|
|
||||||
!!! warning
|
|
||||||
|
|
||||||
`python -m vllm.entrypoints.openai.api_server` is deprecated
|
|
||||||
and may become unsupported in a future release.
|
|
||||||
|
|
||||||
That code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py).
|
|
||||||
|
|
||||||
More details on the API server can be found in the [Online Serving](../serving/online_serving/README.md) document.
|
More details on the API server can be found in the [Online Serving](../serving/online_serving/README.md) document.
|
||||||
|
|
||||||
## V1 Process Architecture
|
## V1 Process Architecture
|
||||||
@@ -88,7 +74,7 @@ The API server process handles HTTP requests (e.g., the OpenAI-compatible API),
|
|||||||
|
|
||||||
By default, there is **1 API server process**, but when data parallelism is used, the API server count automatically scales to match the data parallel size. This can also be manually configured with the `--api-server-count` flag. Each API server connects to **all** engine cores via ZMQ in a many-to-many topology, enabling any API server to route requests to any engine core. Each API server process uses multiple CPU threads for media loading (controlled by `VLLM_MEDIA_LOADING_THREAD_COUNT`, default 8).
|
By default, there is **1 API server process**, but when data parallelism is used, the API server count automatically scales to match the data parallel size. This can also be manually configured with the `--api-server-count` flag. Each API server connects to **all** engine cores via ZMQ in a many-to-many topology, enabling any API server to route requests to any engine core. Each API server process uses multiple CPU threads for media loading (controlled by `VLLM_MEDIA_LOADING_THREAD_COUNT`, default 8).
|
||||||
|
|
||||||
The code can be found in [vllm/entrypoints/openai/api_server.py](../../vllm/entrypoints/openai/api_server.py) and [vllm/v1/utils.py](../../vllm/v1/utils.py).
|
The code can be found in [vllm/entrypoints/launchers/api_server](../../vllm/entrypoints/launchers/api_server) and [vllm/v1/utils.py](../../vllm/v1/utils.py).
|
||||||
|
|
||||||
### Engine Core Process
|
### Engine Core Process
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Endpoint Plugins
|
# Endpoint Plugins
|
||||||
|
|
||||||
Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server without editing `vllm/entrypoints/openai/api_server.py`. Their scope is
|
Endpoint plugins let out-of-tree packages add HTTP routes to the OpenAI compatible API server. Their scope is
|
||||||
the **HTTP surface only** registering routes and optionally per app state used by those routes. A plugin reaches the engine the same way an in-tree serving handler does, through the `EngineClient` it is handed at startup (e.g. `engine_client.collective_rpc(...)`). No new engine access path is introduced.
|
the **HTTP surface only** registering routes and optionally per app state used by those routes. A plugin reaches the engine the same way an in-tree serving handler does, through the `EngineClient` it is handed at startup (e.g. `engine_client.collective_rpc(...)`). No new engine access path is introduced.
|
||||||
|
|
||||||
!!! warning "Security"
|
!!! warning "Security"
|
||||||
|
|||||||
@@ -4,8 +4,7 @@
|
|||||||
NOTE: This API server is used only for demonstrating usage of AsyncEngine
|
NOTE: This API server is used only for demonstrating usage of AsyncEngine
|
||||||
and simple performance benchmarks. It is not intended for production use.
|
and simple performance benchmarks. It is not intended for production use.
|
||||||
For production use, we recommend using our OpenAI compatible server.
|
For production use, we recommend using our OpenAI compatible server.
|
||||||
We are also not going to accept PRs modifying this file, please
|
We are also not going to accept PRs modifying this file.
|
||||||
change `vllm/entrypoints/openai/api_server.py` instead.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -21,7 +20,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
|
|||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||||
from vllm.entrypoints.launcher import serve_http
|
from vllm.entrypoints.launchers.launcher import serve_http
|
||||||
from vllm.entrypoints.serve.utils.api_utils import with_cancellation
|
from vllm.entrypoints.serve.utils.api_utils import with_cancellation
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|||||||
+1
-1
@@ -10,7 +10,7 @@ from unittest.mock import patch
|
|||||||
import pytest
|
import pytest
|
||||||
import zmq
|
import zmq
|
||||||
|
|
||||||
from tests.entrypoints.unit_tests._api_server_spawn_workers import (
|
from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||||
exit_before_report_worker,
|
exit_before_report_worker,
|
||||||
)
|
)
|
||||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||||
+1
-1
@@ -147,7 +147,7 @@ async def test_shutdown_on_engine_failure(tmp_path: Path):
|
|||||||
# dtype, max-len etc set so that this can run in CI
|
# dtype, max-len etc set so that this can run in CI
|
||||||
sys.executable,
|
sys.executable,
|
||||||
"-m",
|
"-m",
|
||||||
"vllm.entrypoints.openai.api_server",
|
"vllm.entrypoints.launchers.api_server.entry",
|
||||||
"--model",
|
"--model",
|
||||||
MODEL_NAME,
|
MODEL_NAME,
|
||||||
"--dtype",
|
"--dtype",
|
||||||
+1
-1
@@ -7,7 +7,7 @@ from ssl import SSLContext
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
|
from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher
|
||||||
|
|
||||||
|
|
||||||
class MockSSLContext(SSLContext):
|
class MockSSLContext(SSLContext):
|
||||||
@@ -14,7 +14,7 @@ import pytest
|
|||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from prometheus_client import CollectorRegistry
|
from prometheus_client import CollectorRegistry
|
||||||
|
|
||||||
from vllm.entrypoints.openai.api_server import build_app
|
from vllm.entrypoints.launchers.api_server.entry import build_app
|
||||||
from vllm.exceptions import (
|
from vllm.exceptions import (
|
||||||
VLLMNotFoundError,
|
VLLMNotFoundError,
|
||||||
VLLMServerError,
|
VLLMServerError,
|
||||||
|
|||||||
+1
-2
@@ -6,11 +6,10 @@ from tempfile import TemporaryDirectory
|
|||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from tests.utils import RemoteOpenAIServer
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.version import __version__ as VLLM_VERSION
|
from vllm.version import __version__ as VLLM_VERSION
|
||||||
|
|
||||||
from ...utils import RemoteOpenAIServer
|
|
||||||
|
|
||||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||||
|
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ import time
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.entrypoints.openai.api_server import (
|
from vllm.entrypoints.launchers.api_server.entry import (
|
||||||
build_async_engine_client_from_engine_args,
|
build_async_engine_client_from_engine_args,
|
||||||
)
|
)
|
||||||
from vllm.inputs import TextPrompt
|
from vllm.inputs import TextPrompt
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Script to test add_lora, remove_lora, pin_lora, list_loras functions.
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||||
from vllm.entrypoints.openai.api_server import (
|
from vllm.entrypoints.launchers.api_server.entry import (
|
||||||
build_async_engine_client_from_engine_args,
|
build_async_engine_client_from_engine_args,
|
||||||
)
|
)
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
|
|||||||
@@ -17,14 +17,14 @@ import pytest
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
|
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
|
||||||
|
|
||||||
from vllm.entrypoints.openai.api_server import (
|
from vllm.entrypoints.launchers.app import build_app
|
||||||
_attach_endpoint_plugins,
|
|
||||||
_init_endpoint_plugins_state,
|
|
||||||
build_app,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||||
from vllm.plugins import load_endpoint_plugins
|
from vllm.plugins import load_endpoint_plugins
|
||||||
from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
|
from vllm.plugins.endpoint_plugins.interface import (
|
||||||
|
EndpointPlugin,
|
||||||
|
attach_endpoint_plugins,
|
||||||
|
init_endpoint_plugins_state,
|
||||||
|
)
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
|
||||||
@@ -157,7 +157,7 @@ def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch)
|
|||||||
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
|
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
_attach_endpoint_plugins(app, ("generate",))
|
attach_endpoint_plugins(app, ("generate",))
|
||||||
|
|
||||||
assert app.state.endpoint_plugins == []
|
assert app.state.endpoint_plugins == []
|
||||||
|
|
||||||
@@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc
|
|||||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||||
|
|
||||||
state = State()
|
state = State()
|
||||||
await _init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
|
await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
|
||||||
|
|
||||||
assert not hasattr(state, "dummy_engine_client")
|
assert not hasattr(state, "dummy_engine_client")
|
||||||
|
|
||||||
@@ -196,7 +196,7 @@ async def test_render_server_attaches_endpoint_plugins_with_no_engine_client(
|
|||||||
for route in app.routes
|
for route in app.routes
|
||||||
)
|
)
|
||||||
|
|
||||||
await _init_endpoint_plugins_state(None, app.state, args)
|
await init_endpoint_plugins_state(None, app.state, args)
|
||||||
|
|
||||||
assert app.state.dummy_engine_client is None
|
assert app.state.dummy_engine_client is None
|
||||||
|
|
||||||
@@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
|
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
|
||||||
await _init_endpoint_plugins_state(fake_engine_client, app.state, args)
|
await init_endpoint_plugins_state(fake_engine_client, app.state, args)
|
||||||
|
|
||||||
assert app.state.dummy_engine_client is fake_engine_client
|
assert app.state.dummy_engine_client is fake_engine_client
|
||||||
|
|
||||||
|
|||||||
@@ -305,7 +305,7 @@ async def run_vllm_async(
|
|||||||
disable_detokenize: bool = False,
|
disable_detokenize: bool = False,
|
||||||
warmup_requests: list[SampleRequest] | None = None,
|
warmup_requests: list[SampleRequest] | None = None,
|
||||||
) -> float:
|
) -> float:
|
||||||
from vllm.entrypoints.openai.api_server import (
|
from vllm.entrypoints.launchers.api_server.entry import (
|
||||||
build_async_engine_client_from_engine_args,
|
build_async_engine_client_from_engine_args,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -3,18 +3,11 @@
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import inspect
|
import inspect
|
||||||
import signal
|
|
||||||
|
|
||||||
import uvloop
|
import uvloop
|
||||||
|
|
||||||
from vllm import envs
|
|
||||||
from vllm.config import VllmConfig
|
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
|
||||||
from vllm.entrypoints.cli.types import CLISubcommand
|
from vllm.entrypoints.cli.types import CLISubcommand
|
||||||
from vllm.entrypoints.openai.api_server import (
|
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
|
||||||
build_and_serve_renderer,
|
|
||||||
setup_server,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.cli_args import (
|
from vllm.entrypoints.openai.cli_args import (
|
||||||
make_arg_parser,
|
make_arg_parser,
|
||||||
validate_parsed_serve_args,
|
validate_parsed_serve_args,
|
||||||
@@ -120,38 +113,3 @@ class LaunchSubcommand(CLISubcommand):
|
|||||||
|
|
||||||
def cmd_init() -> list[CLISubcommand]:
|
def cmd_init() -> list[CLISubcommand]:
|
||||||
return [LaunchSubcommand()]
|
return [LaunchSubcommand()]
|
||||||
|
|
||||||
|
|
||||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
|
||||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
|
||||||
|
|
||||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
|
||||||
# its own signal handlers. Once uvicorn is running it replaces this.
|
|
||||||
def _interrupt_init(*_) -> None:
|
|
||||||
raise KeyboardInterrupt("terminated")
|
|
||||||
|
|
||||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
|
||||||
|
|
||||||
# 1. Socket binding
|
|
||||||
listen_address, sock = setup_server(args, reuse_port=False)
|
|
||||||
|
|
||||||
# 2. Build and serve the API server
|
|
||||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
|
||||||
model_config = engine_args.create_model_config()
|
|
||||||
|
|
||||||
# Render servers preprocess data only — no inference, no quantized kernels.
|
|
||||||
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
|
||||||
model_config.quantization = None
|
|
||||||
|
|
||||||
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
|
||||||
# cache space warning from CpuPlatform.check_and_update_config.
|
|
||||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
|
||||||
|
|
||||||
vllm_config = VllmConfig(model_config=model_config)
|
|
||||||
shutdown_task = await build_and_serve_renderer(
|
|
||||||
vllm_config, listen_address, sock, args
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
await shutdown_task
|
|
||||||
finally:
|
|
||||||
sock.close()
|
|
||||||
|
|||||||
@@ -10,11 +10,9 @@ import uvloop
|
|||||||
import vllm
|
import vllm
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.cli.types import CLISubcommand
|
from vllm.entrypoints.cli.types import CLISubcommand
|
||||||
from vllm.entrypoints.openai.api_server import run_server, setup_server
|
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||||
from vllm.entrypoints.openai.dp_supervisor import (
|
from vllm.entrypoints.openai.dp_supervisor import run_dp_supervisor
|
||||||
run_dp_supervisor,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.usage.usage_lib import UsageContext
|
from vllm.usage.usage_lib import UsageContext
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import warnings
|
||||||
|
from argparse import Namespace
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
|
from starlette.datastructures import State
|
||||||
|
|
||||||
|
from vllm.engine.protocol import EngineClient
|
||||||
|
from vllm.entrypoints.chat_utils import load_chat_template
|
||||||
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
|
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||||
|
from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
|
||||||
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||||
|
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||||
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
|
||||||
|
|
||||||
|
|
||||||
|
async def init_app_state(
|
||||||
|
engine_client: EngineClient,
|
||||||
|
state: State,
|
||||||
|
args: Namespace,
|
||||||
|
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||||
|
) -> None:
|
||||||
|
vllm_config = engine_client.vllm_config
|
||||||
|
|
||||||
|
if args.tool_call_parser is not None:
|
||||||
|
from vllm.parser.metrics import init_parser_metrics
|
||||||
|
|
||||||
|
init_parser_metrics(
|
||||||
|
model_name=cast(str, vllm_config.model_config.served_model_name)
|
||||||
|
)
|
||||||
|
|
||||||
|
if supported_tasks is None:
|
||||||
|
warnings.warn(
|
||||||
|
"The 'supported_tasks' parameter was not provided to "
|
||||||
|
"init_app_state and will be required in a future version. "
|
||||||
|
"Please pass 'supported_tasks' explicitly.",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
|
supported_tasks = FALLBACK_SUPPORTED_TASKS
|
||||||
|
|
||||||
|
if args.served_model_name is not None:
|
||||||
|
served_model_names = args.served_model_name
|
||||||
|
else:
|
||||||
|
served_model_names = [args.model]
|
||||||
|
|
||||||
|
if args.enable_log_requests:
|
||||||
|
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||||
|
else:
|
||||||
|
request_logger = None
|
||||||
|
|
||||||
|
base_model_paths = [
|
||||||
|
BaseModelPath(name=name, model_path=args.model) for name in served_model_names
|
||||||
|
]
|
||||||
|
|
||||||
|
state.engine_client = engine_client
|
||||||
|
state.log_stats = not args.disable_log_stats
|
||||||
|
state.vllm_config = vllm_config
|
||||||
|
state.args = args
|
||||||
|
resolved_chat_template = load_chat_template(args.chat_template)
|
||||||
|
|
||||||
|
# Merge default_mm_loras into the static lora_modules
|
||||||
|
default_mm_loras = (
|
||||||
|
vllm_config.lora_config.default_mm_loras
|
||||||
|
if vllm_config.lora_config is not None
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
|
||||||
|
|
||||||
|
state.openai_serving_models = OpenAIServingModels(
|
||||||
|
engine_client=engine_client,
|
||||||
|
base_model_paths=base_model_paths,
|
||||||
|
lora_modules=lora_modules,
|
||||||
|
)
|
||||||
|
await state.openai_serving_models.init_static_loras()
|
||||||
|
|
||||||
|
state.online_renderer = OnlineRenderer(
|
||||||
|
model_config=engine_client.model_config,
|
||||||
|
renderer=engine_client.renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
|
tool_parser=args.tool_call_parser,
|
||||||
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
log_error_stack=args.log_error_stack,
|
||||||
|
)
|
||||||
|
state.online_renderer.warmup()
|
||||||
|
|
||||||
|
state.online_derenderer = OnlineDerenderer(
|
||||||
|
model_config=engine_client.model_config,
|
||||||
|
renderer=engine_client.renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
|
tool_parser=args.tool_call_parser,
|
||||||
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
log_error_stack=args.log_error_stack,
|
||||||
|
)
|
||||||
|
|
||||||
|
state.serving_tokenization = ServingTokenization(
|
||||||
|
state.openai_serving_models,
|
||||||
|
state.online_renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
)
|
||||||
|
|
||||||
|
if "generate" in supported_tasks:
|
||||||
|
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||||
|
|
||||||
|
await init_generate_state(
|
||||||
|
engine_client, state, args, request_logger, supported_tasks
|
||||||
|
)
|
||||||
|
|
||||||
|
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||||
|
|
||||||
|
init_scale_out_state(state, args, engine_client, request_logger)
|
||||||
|
|
||||||
|
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||||
|
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
||||||
|
|
||||||
|
init_speech_to_text_state(
|
||||||
|
engine_client, state, args, request_logger, supported_tasks
|
||||||
|
)
|
||||||
|
|
||||||
|
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||||
|
from vllm.entrypoints.pooling.factories import init_pooling_state
|
||||||
|
|
||||||
|
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||||
|
|
||||||
|
await init_endpoint_plugins_state(engine_client, state, args)
|
||||||
|
|
||||||
|
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||||
|
state.server_load_metrics = 0
|
||||||
@@ -0,0 +1,229 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import asyncio
|
||||||
|
import multiprocessing
|
||||||
|
import multiprocessing.forkserver as forkserver
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
|
import tempfile
|
||||||
|
from argparse import Namespace
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
|
from vllm.engine.protocol import EngineClient
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.reasoning import ReasoningParserManager
|
||||||
|
from vllm.tool_parsers import ToolParserManager
|
||||||
|
from vllm.usage.usage_lib import UsageContext
|
||||||
|
from vllm.utils.system_utils import decorate_logs
|
||||||
|
|
||||||
|
from ..app import build_app
|
||||||
|
from ..launcher import serve_http, setup_server
|
||||||
|
from ..utils.server_utils import get_uvicorn_log_config
|
||||||
|
from .app_state import init_app_state
|
||||||
|
|
||||||
|
prometheus_multiproc_dir: tempfile.TemporaryDirectory
|
||||||
|
|
||||||
|
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
|
||||||
|
logger = init_logger("vllm.entrypoints.launchers.api_server.entry")
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def build_async_engine_client(
|
||||||
|
args: Namespace,
|
||||||
|
*,
|
||||||
|
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||||
|
client_config: dict[str, Any] | None = None,
|
||||||
|
) -> AsyncIterator[EngineClient]:
|
||||||
|
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
|
||||||
|
# The executor is expected to be mp.
|
||||||
|
# Pre-import heavy modules in the forkserver process
|
||||||
|
logger.debug("Setup forkserver with pre-imports")
|
||||||
|
multiprocessing.set_start_method("forkserver")
|
||||||
|
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
|
||||||
|
forkserver.ensure_running()
|
||||||
|
logger.debug("Forkserver setup complete!")
|
||||||
|
|
||||||
|
# Context manager to handle engine_client lifecycle
|
||||||
|
# Ensures everything is shutdown and cleaned up on error/exit
|
||||||
|
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||||
|
if client_config:
|
||||||
|
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||||
|
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||||
|
|
||||||
|
async with build_async_engine_client_from_engine_args(
|
||||||
|
engine_args,
|
||||||
|
usage_context=usage_context,
|
||||||
|
client_config=client_config,
|
||||||
|
) as engine:
|
||||||
|
yield engine
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def build_async_engine_client_from_engine_args(
|
||||||
|
engine_args: AsyncEngineArgs,
|
||||||
|
*,
|
||||||
|
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||||
|
client_config: dict[str, Any] | None = None,
|
||||||
|
) -> AsyncIterator[EngineClient]:
|
||||||
|
"""
|
||||||
|
Create EngineClient, either:
|
||||||
|
- in-process using the AsyncLLMEngine Directly
|
||||||
|
- multiprocess using AsyncLLMEngine RPC
|
||||||
|
|
||||||
|
Returns the Client or None if the creation failed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Create the EngineConfig (determines if we can use V1).
|
||||||
|
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||||
|
|
||||||
|
from vllm.v1.engine.async_llm import AsyncLLM
|
||||||
|
|
||||||
|
async_llm: AsyncLLM | None = None
|
||||||
|
|
||||||
|
# Don't mutate the input client_config
|
||||||
|
client_config = dict(client_config) if client_config else {}
|
||||||
|
client_count = client_config.pop("client_count", 1)
|
||||||
|
client_index = client_config.pop("client_index", 0)
|
||||||
|
|
||||||
|
try:
|
||||||
|
async_llm = AsyncLLM.from_vllm_config(
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
usage_context=usage_context,
|
||||||
|
enable_log_requests=engine_args.enable_log_requests,
|
||||||
|
aggregate_engine_logging=engine_args.aggregate_engine_logging,
|
||||||
|
disable_log_stats=engine_args.disable_log_stats,
|
||||||
|
client_addresses=client_config,
|
||||||
|
client_count=client_count,
|
||||||
|
client_index=client_index,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Don't keep the dummy data in memory
|
||||||
|
assert async_llm is not None
|
||||||
|
await async_llm.reset_mm_cache()
|
||||||
|
|
||||||
|
yield async_llm
|
||||||
|
finally:
|
||||||
|
if async_llm:
|
||||||
|
async_llm.shutdown(timeout=vllm_config.shutdown_timeout)
|
||||||
|
|
||||||
|
|
||||||
|
async def build_and_serve(
|
||||||
|
engine_client: EngineClient,
|
||||||
|
listen_address: str,
|
||||||
|
sock: socket.socket,
|
||||||
|
args: Namespace,
|
||||||
|
**uvicorn_kwargs,
|
||||||
|
) -> asyncio.Task:
|
||||||
|
"""Build FastAPI app, initialize state, and start serving.
|
||||||
|
|
||||||
|
Returns the shutdown task for the caller to await.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Get uvicorn log config (from file or with endpoint filter)
|
||||||
|
log_config = get_uvicorn_log_config(args)
|
||||||
|
if log_config is not None:
|
||||||
|
uvicorn_kwargs["log_config"] = log_config
|
||||||
|
|
||||||
|
supported_tasks = await engine_client.get_supported_tasks()
|
||||||
|
model_config = engine_client.model_config
|
||||||
|
|
||||||
|
logger.info("Supported tasks: %s", supported_tasks)
|
||||||
|
app = build_app(args, supported_tasks, model_config)
|
||||||
|
await init_app_state(engine_client, app.state, args, supported_tasks)
|
||||||
|
|
||||||
|
logger.info("Starting vLLM server on %s", listen_address)
|
||||||
|
|
||||||
|
return await serve_http(
|
||||||
|
app,
|
||||||
|
sock=sock,
|
||||||
|
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||||
|
host=args.host,
|
||||||
|
port=args.port,
|
||||||
|
log_level=args.uvicorn_log_level,
|
||||||
|
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||||
|
# no access log will be output.
|
||||||
|
access_log=not args.disable_uvicorn_access_log,
|
||||||
|
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||||
|
ssl_keyfile=args.ssl_keyfile,
|
||||||
|
ssl_certfile=args.ssl_certfile,
|
||||||
|
ssl_ca_certs=args.ssl_ca_certs,
|
||||||
|
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||||
|
ssl_ciphers=args.ssl_ciphers,
|
||||||
|
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||||
|
h11_max_header_count=args.h11_max_header_count,
|
||||||
|
**uvicorn_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||||
|
"""Run a single-worker API server."""
|
||||||
|
|
||||||
|
decorate_logs("APIServer", skip_if_decorated=True)
|
||||||
|
|
||||||
|
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
||||||
|
# own signal handlers. Once uvicorn is running it replaces this.
|
||||||
|
def _interrupt_init(*_) -> None:
|
||||||
|
raise KeyboardInterrupt("terminated")
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||||
|
|
||||||
|
listen_address, sock = setup_server(args, reuse_port=False)
|
||||||
|
await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_server_worker(
|
||||||
|
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
||||||
|
) -> None:
|
||||||
|
"""Run a single API server worker."""
|
||||||
|
|
||||||
|
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||||
|
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||||
|
|
||||||
|
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||||
|
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||||
|
|
||||||
|
async with build_async_engine_client(
|
||||||
|
args,
|
||||||
|
client_config=client_config,
|
||||||
|
) as engine_client:
|
||||||
|
shutdown_task = await build_and_serve(
|
||||||
|
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
||||||
|
)
|
||||||
|
# NB: Await server shutdown only after the backend context is exited
|
||||||
|
try:
|
||||||
|
await shutdown_task
|
||||||
|
finally:
|
||||||
|
sock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
import uvloop
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.cli_args import (
|
||||||
|
make_arg_parser,
|
||||||
|
validate_parsed_serve_args,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
# NOTE(simon):
|
||||||
|
# This section should be in sync with vllm/entrypoints/cli/main.py for CLI
|
||||||
|
# entrypoints.
|
||||||
|
cli_env_setup()
|
||||||
|
parser = FlexibleArgumentParser(
|
||||||
|
description="vLLM OpenAI-Compatible RESTful API server."
|
||||||
|
)
|
||||||
|
parser = make_arg_parser(parser)
|
||||||
|
args = parser.parse_args()
|
||||||
|
validate_parsed_serve_args(args)
|
||||||
|
|
||||||
|
uvloop.run(run_server(args))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import warnings
|
||||||
|
from argparse import Namespace
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
|
||||||
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||||
|
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||||
|
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||||
|
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
|
||||||
|
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||||
|
|
||||||
|
from .api_server.routers import register_api_routers
|
||||||
|
from .utils.server_utils import lifespan
|
||||||
|
|
||||||
|
|
||||||
|
def build_app(
|
||||||
|
args: Namespace,
|
||||||
|
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
||||||
|
model_config: ModelConfig | None = None,
|
||||||
|
) -> FastAPI:
|
||||||
|
if supported_tasks is None:
|
||||||
|
warnings.warn(
|
||||||
|
"The 'supported_tasks' parameter was not provided to "
|
||||||
|
"build_app and will be required in a future version. "
|
||||||
|
"Defaulting to ('generate',).",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
|
supported_tasks = FALLBACK_SUPPORTED_TASKS
|
||||||
|
|
||||||
|
if args.disable_fastapi_docs:
|
||||||
|
app = FastAPI(
|
||||||
|
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
||||||
|
)
|
||||||
|
elif args.enable_offline_docs:
|
||||||
|
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||||
|
else:
|
||||||
|
app = FastAPI(lifespan=lifespan)
|
||||||
|
app.state.args = args
|
||||||
|
app.root_path = args.root_path
|
||||||
|
|
||||||
|
register_api_routers(args, app, supported_tasks, model_config)
|
||||||
|
|
||||||
|
# Endpoint plugins are attached last so their routes are registered after all core
|
||||||
|
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||||
|
# the `render` task still gets its routes registered. It receives
|
||||||
|
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||||
|
attach_endpoint_plugins(app, supported_tasks)
|
||||||
|
|
||||||
|
init_exception_handler(app)
|
||||||
|
init_entrypoints_middleware(args, app, supported_tasks)
|
||||||
|
app = sagemaker_standards_bootstrap(app)
|
||||||
|
return app
|
||||||
@@ -14,13 +14,23 @@ from fastapi import FastAPI
|
|||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.serve.utils.constants import (
|
from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher
|
||||||
|
from vllm.entrypoints.serve.utils.api_utils import (
|
||||||
|
log_non_default_args,
|
||||||
|
log_version_and_model,
|
||||||
|
)
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.reasoning import ReasoningParserManager
|
||||||
|
from vllm.tool_parsers import ToolParserManager
|
||||||
|
from vllm.tracing import instrument
|
||||||
|
from vllm.utils.network_utils import find_process_using_port, is_valid_ipv6_address
|
||||||
|
from vllm.utils.system_utils import set_ulimit
|
||||||
|
from vllm.version import __version__ as VLLM_VERSION
|
||||||
|
|
||||||
|
from .utils.constants import (
|
||||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
|
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.utils.network_utils import find_process_using_port
|
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -190,3 +200,83 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
|
|||||||
engine_errored = engine.errored and not engine.is_running
|
engine_errored = engine.errored and not engine.is_running
|
||||||
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
|
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
|
||||||
server.should_exit = True
|
server.should_exit = True
|
||||||
|
|
||||||
|
|
||||||
|
def create_server_socket(
|
||||||
|
addr: tuple[str, int],
|
||||||
|
*,
|
||||||
|
reuse_port: bool,
|
||||||
|
) -> socket.socket:
|
||||||
|
family = socket.AF_INET
|
||||||
|
if is_valid_ipv6_address(addr[0]):
|
||||||
|
family = socket.AF_INET6
|
||||||
|
|
||||||
|
sock = socket.socket(family=family, type=socket.SOCK_STREAM)
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
if reuse_port:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
||||||
|
sock.bind(addr)
|
||||||
|
|
||||||
|
return sock
|
||||||
|
|
||||||
|
|
||||||
|
def create_server_unix_socket(path: str) -> socket.socket:
|
||||||
|
sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
|
||||||
|
sock.bind(path)
|
||||||
|
return sock
|
||||||
|
|
||||||
|
|
||||||
|
def validate_api_server_args(args):
|
||||||
|
valid_tool_parses = ToolParserManager.list_registered()
|
||||||
|
if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
|
||||||
|
raise KeyError(
|
||||||
|
f"invalid tool call parser: {args.tool_call_parser} "
|
||||||
|
f"(chose from {{ {','.join(valid_tool_parses)} }})"
|
||||||
|
)
|
||||||
|
|
||||||
|
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
||||||
|
if (
|
||||||
|
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
||||||
|
) and reasoning_parser not in valid_reasoning_parsers:
|
||||||
|
raise KeyError(
|
||||||
|
f"invalid reasoning parser: {reasoning_parser} "
|
||||||
|
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@instrument(span_name="API server setup")
|
||||||
|
def setup_server(args, *, reuse_port: bool):
|
||||||
|
"""Validate API server args and create the server socket."""
|
||||||
|
|
||||||
|
log_version_and_model(logger, VLLM_VERSION, args.model)
|
||||||
|
log_non_default_args(args)
|
||||||
|
|
||||||
|
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||||
|
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||||
|
|
||||||
|
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
||||||
|
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
||||||
|
|
||||||
|
validate_api_server_args(args)
|
||||||
|
|
||||||
|
# workaround to make sure that we bind the port before the engine is set up.
|
||||||
|
# This avoids race conditions with ray.
|
||||||
|
# see https://github.com/vllm-project/vllm/issues/8204
|
||||||
|
if args.uds:
|
||||||
|
sock = create_server_unix_socket(args.uds)
|
||||||
|
else:
|
||||||
|
sock_addr = (args.host or "", args.port)
|
||||||
|
sock = create_server_socket(sock_addr, reuse_port=reuse_port)
|
||||||
|
|
||||||
|
# workaround to avoid footguns where uvicorn drops requests with too
|
||||||
|
# many concurrent requests active
|
||||||
|
set_ulimit()
|
||||||
|
|
||||||
|
if args.uds:
|
||||||
|
listen_address = f"unix:{args.uds}"
|
||||||
|
else:
|
||||||
|
addr, port = sock_addr
|
||||||
|
is_ssl = args.ssl_keyfile and args.ssl_certfile
|
||||||
|
host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
|
||||||
|
listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
|
||||||
|
return listen_address, sock
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from argparse import Namespace
|
||||||
|
|
||||||
|
from starlette.datastructures import State
|
||||||
|
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.entrypoints.chat_utils import load_chat_template
|
||||||
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
|
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||||
|
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||||
|
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||||
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||||
|
from vllm.renderers import renderer_from_config
|
||||||
|
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||||
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|
||||||
|
|
||||||
|
async def init_render_app_state(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
state: State,
|
||||||
|
args: Namespace,
|
||||||
|
) -> None:
|
||||||
|
"""Initialise FastAPI app state for a CPU-only render server.
|
||||||
|
|
||||||
|
Unlike :func:`init_app_state` this function does not require an
|
||||||
|
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
||||||
|
preprocessing pipeline (renderer, input_processor)
|
||||||
|
directly from the :class:`~vllm.config.VllmConfig`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
served_model_names = args.served_model_name or [args.model]
|
||||||
|
model_registry = OpenAIModelRegistry(
|
||||||
|
model_config=vllm_config.model_config,
|
||||||
|
base_model_paths=[
|
||||||
|
BaseModelPath(name=name, model_path=args.model)
|
||||||
|
for name in served_model_names
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.enable_log_requests:
|
||||||
|
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||||
|
else:
|
||||||
|
request_logger = None
|
||||||
|
|
||||||
|
renderer = renderer_from_config(vllm_config)
|
||||||
|
resolved_chat_template = load_chat_template(args.chat_template)
|
||||||
|
|
||||||
|
state.online_renderer = OnlineRenderer(
|
||||||
|
model_config=vllm_config.model_config,
|
||||||
|
renderer=renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
|
tool_parser=args.tool_call_parser,
|
||||||
|
reasoning_parser=args.reasoning_parser,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
log_error_stack=args.log_error_stack,
|
||||||
|
)
|
||||||
|
state.online_renderer.warmup()
|
||||||
|
|
||||||
|
state.online_derenderer = OnlineDerenderer(
|
||||||
|
model_config=vllm_config.model_config,
|
||||||
|
renderer=renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
|
tool_parser=args.tool_call_parser,
|
||||||
|
reasoning_parser=args.reasoning_parser,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
log_error_stack=args.log_error_stack,
|
||||||
|
)
|
||||||
|
|
||||||
|
state.openai_serving_models = model_registry
|
||||||
|
state.serving_tokenization = ServingTokenization(
|
||||||
|
model_registry,
|
||||||
|
state.online_renderer,
|
||||||
|
request_logger=request_logger,
|
||||||
|
chat_template=resolved_chat_template,
|
||||||
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
|
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||||
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
)
|
||||||
|
|
||||||
|
init_render_state(state, request_logger)
|
||||||
|
|
||||||
|
state.vllm_config = vllm_config
|
||||||
|
# Disable stats logging — there is no engine to poll.
|
||||||
|
state.log_stats = False
|
||||||
|
state.engine_client = None
|
||||||
|
state.args = args
|
||||||
|
state.enable_server_load_tracking = False
|
||||||
|
state.server_load_metrics = 0
|
||||||
|
|
||||||
|
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||||
|
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||||
|
await init_endpoint_plugins_state(None, state, args)
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
|
from argparse import Namespace
|
||||||
|
|
||||||
|
from vllm import AsyncEngineArgs, envs
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
from ..app import build_app
|
||||||
|
from ..launcher import serve_http, setup_server
|
||||||
|
from ..utils.server_utils import get_uvicorn_log_config
|
||||||
|
from .app_state import init_render_app_state
|
||||||
|
|
||||||
|
logger = init_logger("vllm.entrypoints.launchers.render.entry")
|
||||||
|
|
||||||
|
|
||||||
|
async def build_and_serve_renderer(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
listen_address: str,
|
||||||
|
sock: socket.socket,
|
||||||
|
args: Namespace,
|
||||||
|
**uvicorn_kwargs,
|
||||||
|
) -> asyncio.Task:
|
||||||
|
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
||||||
|
start serving.
|
||||||
|
|
||||||
|
Returns the shutdown task for the caller to await.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Get uvicorn log config (from file or with endpoint filter)
|
||||||
|
log_config = get_uvicorn_log_config(args)
|
||||||
|
if log_config is not None:
|
||||||
|
uvicorn_kwargs["log_config"] = log_config
|
||||||
|
|
||||||
|
app = build_app(args, ("render",))
|
||||||
|
await init_render_app_state(vllm_config, app.state, args)
|
||||||
|
|
||||||
|
logger.info("Starting vLLM server on %s", listen_address)
|
||||||
|
|
||||||
|
return await serve_http(
|
||||||
|
app,
|
||||||
|
sock=sock,
|
||||||
|
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||||
|
host=args.host,
|
||||||
|
port=args.port,
|
||||||
|
log_level=args.uvicorn_log_level,
|
||||||
|
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||||
|
# no access log will be output.
|
||||||
|
access_log=not args.disable_uvicorn_access_log,
|
||||||
|
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||||
|
ssl_keyfile=args.ssl_keyfile,
|
||||||
|
ssl_certfile=args.ssl_certfile,
|
||||||
|
ssl_ca_certs=args.ssl_ca_certs,
|
||||||
|
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||||
|
ssl_ciphers=args.ssl_ciphers,
|
||||||
|
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||||
|
h11_max_header_count=args.h11_max_header_count,
|
||||||
|
**uvicorn_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||||
|
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||||
|
|
||||||
|
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
||||||
|
# its own signal handlers. Once uvicorn is running it replaces this.
|
||||||
|
def _interrupt_init(*_) -> None:
|
||||||
|
raise KeyboardInterrupt("terminated")
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||||
|
|
||||||
|
# 1. Socket binding
|
||||||
|
listen_address, sock = setup_server(args, reuse_port=False)
|
||||||
|
|
||||||
|
# 2. Build and serve the API server
|
||||||
|
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||||
|
model_config = engine_args.create_model_config()
|
||||||
|
|
||||||
|
# Render servers preprocess data only — no inference, no quantized kernels.
|
||||||
|
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
||||||
|
model_config.quantization = None
|
||||||
|
|
||||||
|
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
||||||
|
# cache space warning from CpuPlatform.check_and_update_config.
|
||||||
|
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||||
|
|
||||||
|
vllm_config = VllmConfig(model_config=model_config)
|
||||||
|
shutdown_task = await build_and_serve_renderer(
|
||||||
|
vllm_config, listen_address, sock, args
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await shutdown_task
|
||||||
|
finally:
|
||||||
|
sock.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import uvloop
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai.cli_args import (
|
||||||
|
make_arg_parser,
|
||||||
|
validate_parsed_serve_args,
|
||||||
|
)
|
||||||
|
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
cli_env_setup()
|
||||||
|
parser = FlexibleArgumentParser(
|
||||||
|
description="Starts a GPU-less rendering server "
|
||||||
|
"for preprocessing and postprocessing only"
|
||||||
|
)
|
||||||
|
parser = make_arg_parser(parser)
|
||||||
|
args = parser.parse_args()
|
||||||
|
validate_parsed_serve_args(args)
|
||||||
|
|
||||||
|
uvloop.run(run_launch_fastapi(args))
|
||||||
-5
@@ -1,12 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""
|
|
||||||
Shared constants for vLLM entrypoints.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# HTTP header limits for h11 parser
|
# HTTP header limits for h11 parser
|
||||||
# These constants help mitigate header abuse attacks
|
# These constants help mitigate header abuse attacks
|
||||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
|
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
|
||||||
H11_MAX_HEADER_COUNT_DEFAULT = 256
|
H11_MAX_HEADER_COUNT_DEFAULT = 256
|
||||||
|
|
||||||
MCP_PREFIX = "mcp_"
|
|
||||||
+1
-1
@@ -12,7 +12,7 @@ from vllm.engine.protocol import EngineClient
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils.gc_utils import freeze_gc_heap
|
from vllm.utils.gc_utils import freeze_gc_heap
|
||||||
|
|
||||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||||
@@ -1,670 +1,59 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import asyncio
|
|
||||||
import multiprocessing
|
|
||||||
import multiprocessing.forkserver as forkserver
|
|
||||||
import os
|
|
||||||
import signal
|
|
||||||
import socket
|
|
||||||
import tempfile
|
|
||||||
import warnings
|
import warnings
|
||||||
from argparse import Namespace
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from contextlib import asynccontextmanager
|
|
||||||
from typing import Any, cast
|
|
||||||
|
|
||||||
import uvloop
|
from vllm.entrypoints.launchers.api_server.app_state import init_app_state
|
||||||
from fastapi import FastAPI
|
from vllm.entrypoints.launchers.api_server.entry import (
|
||||||
from starlette.datastructures import State
|
build_and_serve,
|
||||||
|
build_async_engine_client,
|
||||||
import vllm.envs as envs
|
build_async_engine_client_from_engine_args,
|
||||||
from vllm.config import ModelConfig, VllmConfig
|
run_server,
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
run_server_worker,
|
||||||
from vllm.engine.protocol import EngineClient
|
)
|
||||||
from vllm.entrypoints.chat_utils import load_chat_template
|
|
||||||
from vllm.entrypoints.launcher import serve_http
|
|
||||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
from vllm.entrypoints.launchers.app import build_app
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.launchers.launcher import (
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
create_server_socket,
|
||||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
create_server_unix_socket,
|
||||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
setup_server,
|
||||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
validate_api_server_args,
|
||||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
|
||||||
from vllm.entrypoints.serve.utils.api_utils import (
|
|
||||||
cli_env_setup,
|
|
||||||
log_non_default_args,
|
|
||||||
log_version_and_model,
|
|
||||||
process_lora_modules,
|
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
|
||||||
from vllm.entrypoints.serve.utils.server_utils import (
|
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
|
||||||
get_uvicorn_log_config,
|
|
||||||
lifespan,
|
warnings.warn(
|
||||||
|
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
|
||||||
|
"unsupported in a future version. Use the corresponding function from "
|
||||||
|
"`vllm.entrypoints.launchers` instead.",
|
||||||
|
DeprecationWarning,
|
||||||
|
stacklevel=1,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
|
||||||
from vllm.reasoning import ReasoningParserManager
|
|
||||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
|
||||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
|
||||||
from vllm.tool_parsers import ToolParserManager
|
|
||||||
from vllm.tracing import instrument
|
|
||||||
from vllm.usage.usage_lib import UsageContext
|
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
|
||||||
from vllm.utils.network_utils import is_valid_ipv6_address
|
|
||||||
from vllm.utils.system_utils import decorate_logs, set_ulimit
|
|
||||||
from vllm.version import __version__ as VLLM_VERSION
|
|
||||||
|
|
||||||
prometheus_multiproc_dir: tempfile.TemporaryDirectory
|
|
||||||
|
|
||||||
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
|
|
||||||
logger = init_logger("vllm.entrypoints.openai.api_server")
|
|
||||||
|
|
||||||
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
|
|
||||||
|
|
||||||
|
|
||||||
def _attach_endpoint_plugins(
|
|
||||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
|
||||||
) -> None:
|
|
||||||
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
|
|
||||||
|
|
||||||
Attached last after all core routers. This is so endpoint plugin routes can
|
|
||||||
shadow core routes with the same path (see `EndpointPlugin.attach_router`
|
|
||||||
docstring). No-ops when no plugins are discovered/allowlisted.
|
|
||||||
"""
|
|
||||||
from vllm.plugins import load_endpoint_plugins
|
|
||||||
|
|
||||||
endpoint_plugins = load_endpoint_plugins(supported_tasks)
|
|
||||||
for plugin in endpoint_plugins:
|
|
||||||
plugin.attach_router(app)
|
|
||||||
app.state.endpoint_plugins = endpoint_plugins
|
|
||||||
|
|
||||||
|
|
||||||
async def _init_endpoint_plugins_state(
|
|
||||||
engine_client: EngineClient | None, state: State, args: Namespace
|
|
||||||
) -> None:
|
|
||||||
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
|
|
||||||
|
|
||||||
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
|
|
||||||
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
|
|
||||||
build their own bare `State` without going through `build_app`. As a result
|
|
||||||
`endpoint_plugins` may be absent and are treated that the same as "none attached".
|
|
||||||
|
|
||||||
`engine_client` is `None` for the CPU only render server which has no
|
|
||||||
engine (see `init_render_app_state`). Plugins must handle a `None`
|
|
||||||
`engine_client` themselves (see `EndpointPlugin.init_state`).
|
|
||||||
"""
|
|
||||||
for plugin in getattr(state, "endpoint_plugins", []):
|
|
||||||
await plugin.init_state(engine_client, state, args)
|
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
|
||||||
async def build_async_engine_client(
|
|
||||||
args: Namespace,
|
|
||||||
*,
|
|
||||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
|
||||||
client_config: dict[str, Any] | None = None,
|
|
||||||
) -> AsyncIterator[EngineClient]:
|
|
||||||
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
|
|
||||||
# The executor is expected to be mp.
|
|
||||||
# Pre-import heavy modules in the forkserver process
|
|
||||||
logger.debug("Setup forkserver with pre-imports")
|
|
||||||
multiprocessing.set_start_method("forkserver")
|
|
||||||
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
|
|
||||||
forkserver.ensure_running()
|
|
||||||
logger.debug("Forkserver setup complete!")
|
|
||||||
|
|
||||||
# Context manager to handle engine_client lifecycle
|
|
||||||
# Ensures everything is shutdown and cleaned up on error/exit
|
|
||||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
|
||||||
if client_config:
|
|
||||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
|
||||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
|
||||||
|
|
||||||
async with build_async_engine_client_from_engine_args(
|
|
||||||
engine_args,
|
|
||||||
usage_context=usage_context,
|
|
||||||
client_config=client_config,
|
|
||||||
) as engine:
|
|
||||||
yield engine
|
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
|
||||||
async def build_async_engine_client_from_engine_args(
|
|
||||||
engine_args: AsyncEngineArgs,
|
|
||||||
*,
|
|
||||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
|
||||||
client_config: dict[str, Any] | None = None,
|
|
||||||
) -> AsyncIterator[EngineClient]:
|
|
||||||
"""
|
|
||||||
Create EngineClient, either:
|
|
||||||
- in-process using the AsyncLLMEngine Directly
|
|
||||||
- multiprocess using AsyncLLMEngine RPC
|
|
||||||
|
|
||||||
Returns the Client or None if the creation failed.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Create the EngineConfig (determines if we can use V1).
|
|
||||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
|
||||||
|
|
||||||
from vllm.v1.engine.async_llm import AsyncLLM
|
|
||||||
|
|
||||||
async_llm: AsyncLLM | None = None
|
|
||||||
|
|
||||||
# Don't mutate the input client_config
|
|
||||||
client_config = dict(client_config) if client_config else {}
|
|
||||||
client_count = client_config.pop("client_count", 1)
|
|
||||||
client_index = client_config.pop("client_index", 0)
|
|
||||||
|
|
||||||
try:
|
|
||||||
async_llm = AsyncLLM.from_vllm_config(
|
|
||||||
vllm_config=vllm_config,
|
|
||||||
usage_context=usage_context,
|
|
||||||
enable_log_requests=engine_args.enable_log_requests,
|
|
||||||
aggregate_engine_logging=engine_args.aggregate_engine_logging,
|
|
||||||
disable_log_stats=engine_args.disable_log_stats,
|
|
||||||
client_addresses=client_config,
|
|
||||||
client_count=client_count,
|
|
||||||
client_index=client_index,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Don't keep the dummy data in memory
|
|
||||||
assert async_llm is not None
|
|
||||||
await async_llm.reset_mm_cache()
|
|
||||||
|
|
||||||
yield async_llm
|
|
||||||
finally:
|
|
||||||
if async_llm:
|
|
||||||
async_llm.shutdown(timeout=vllm_config.shutdown_timeout)
|
|
||||||
|
|
||||||
|
|
||||||
def build_app(
|
|
||||||
args: Namespace,
|
|
||||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
|
||||||
model_config: ModelConfig | None = None,
|
|
||||||
) -> FastAPI:
|
|
||||||
if supported_tasks is None:
|
|
||||||
warnings.warn(
|
|
||||||
"The 'supported_tasks' parameter was not provided to "
|
|
||||||
"build_app and will be required in a future version. "
|
|
||||||
"Defaulting to ('generate',).",
|
|
||||||
DeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
supported_tasks = _FALLBACK_SUPPORTED_TASKS
|
|
||||||
|
|
||||||
if args.disable_fastapi_docs:
|
|
||||||
app = FastAPI(
|
|
||||||
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
|
||||||
)
|
|
||||||
elif args.enable_offline_docs:
|
|
||||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
|
||||||
else:
|
|
||||||
app = FastAPI(lifespan=lifespan)
|
|
||||||
app.state.args = args
|
|
||||||
app.root_path = args.root_path
|
|
||||||
|
|
||||||
register_api_routers(args, app, supported_tasks, model_config)
|
|
||||||
|
|
||||||
# Endpoint plugins are attached last so their routes are registered after all core
|
|
||||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
|
||||||
# the `render` task still gets its routes registered. It receives
|
|
||||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
|
||||||
_attach_endpoint_plugins(app, supported_tasks)
|
|
||||||
|
|
||||||
init_exception_handler(app)
|
|
||||||
init_entrypoints_middleware(args, app, supported_tasks)
|
|
||||||
app = sagemaker_standards_bootstrap(app)
|
|
||||||
return app
|
|
||||||
|
|
||||||
|
|
||||||
async def init_app_state(
|
|
||||||
engine_client: EngineClient,
|
|
||||||
state: State,
|
|
||||||
args: Namespace,
|
|
||||||
supported_tasks: tuple["SupportedTask", ...] | None = None,
|
|
||||||
) -> None:
|
|
||||||
vllm_config = engine_client.vllm_config
|
|
||||||
|
|
||||||
if args.tool_call_parser is not None:
|
|
||||||
from vllm.parser.metrics import init_parser_metrics
|
|
||||||
|
|
||||||
init_parser_metrics(
|
|
||||||
model_name=cast(str, vllm_config.model_config.served_model_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
if supported_tasks is None:
|
|
||||||
warnings.warn(
|
|
||||||
"The 'supported_tasks' parameter was not provided to "
|
|
||||||
"init_app_state and will be required in a future version. "
|
|
||||||
"Please pass 'supported_tasks' explicitly.",
|
|
||||||
DeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
supported_tasks = _FALLBACK_SUPPORTED_TASKS
|
|
||||||
|
|
||||||
if args.served_model_name is not None:
|
|
||||||
served_model_names = args.served_model_name
|
|
||||||
else:
|
|
||||||
served_model_names = [args.model]
|
|
||||||
|
|
||||||
if args.enable_log_requests:
|
|
||||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
|
||||||
else:
|
|
||||||
request_logger = None
|
|
||||||
|
|
||||||
base_model_paths = [
|
|
||||||
BaseModelPath(name=name, model_path=args.model) for name in served_model_names
|
|
||||||
]
|
|
||||||
|
|
||||||
state.engine_client = engine_client
|
|
||||||
state.log_stats = not args.disable_log_stats
|
|
||||||
state.vllm_config = vllm_config
|
|
||||||
state.args = args
|
|
||||||
resolved_chat_template = load_chat_template(args.chat_template)
|
|
||||||
|
|
||||||
# Merge default_mm_loras into the static lora_modules
|
|
||||||
default_mm_loras = (
|
|
||||||
vllm_config.lora_config.default_mm_loras
|
|
||||||
if vllm_config.lora_config is not None
|
|
||||||
else {}
|
|
||||||
)
|
|
||||||
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
|
|
||||||
|
|
||||||
state.openai_serving_models = OpenAIServingModels(
|
|
||||||
engine_client=engine_client,
|
|
||||||
base_model_paths=base_model_paths,
|
|
||||||
lora_modules=lora_modules,
|
|
||||||
)
|
|
||||||
await state.openai_serving_models.init_static_loras()
|
|
||||||
|
|
||||||
state.online_renderer = OnlineRenderer(
|
|
||||||
model_config=engine_client.model_config,
|
|
||||||
renderer=engine_client.renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
|
||||||
tool_parser=args.tool_call_parser,
|
|
||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
log_error_stack=args.log_error_stack,
|
|
||||||
)
|
|
||||||
state.online_renderer.warmup()
|
|
||||||
|
|
||||||
state.online_derenderer = OnlineDerenderer(
|
|
||||||
model_config=engine_client.model_config,
|
|
||||||
renderer=engine_client.renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
|
||||||
tool_parser=args.tool_call_parser,
|
|
||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
log_error_stack=args.log_error_stack,
|
|
||||||
)
|
|
||||||
|
|
||||||
state.serving_tokenization = ServingTokenization(
|
|
||||||
state.openai_serving_models,
|
|
||||||
state.online_renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
)
|
|
||||||
|
|
||||||
if "generate" in supported_tasks:
|
|
||||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
|
||||||
|
|
||||||
await init_generate_state(
|
|
||||||
engine_client, state, args, request_logger, supported_tasks
|
|
||||||
)
|
|
||||||
|
|
||||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
|
||||||
|
|
||||||
init_scale_out_state(state, args, engine_client, request_logger)
|
|
||||||
|
|
||||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
|
||||||
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
|
|
||||||
|
|
||||||
init_speech_to_text_state(
|
|
||||||
engine_client, state, args, request_logger, supported_tasks
|
|
||||||
)
|
|
||||||
|
|
||||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
|
||||||
from vllm.entrypoints.pooling.factories import init_pooling_state
|
|
||||||
|
|
||||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
|
||||||
|
|
||||||
await _init_endpoint_plugins_state(engine_client, state, args)
|
|
||||||
|
|
||||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
|
||||||
state.server_load_metrics = 0
|
|
||||||
|
|
||||||
|
|
||||||
async def init_render_app_state(
|
|
||||||
vllm_config: VllmConfig,
|
|
||||||
state: State,
|
|
||||||
args: Namespace,
|
|
||||||
) -> None:
|
|
||||||
"""Initialise FastAPI app state for a CPU-only render server.
|
|
||||||
|
|
||||||
Unlike :func:`init_app_state` this function does not require an
|
|
||||||
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
|
||||||
preprocessing pipeline (renderer, input_processor)
|
|
||||||
directly from the :class:`~vllm.config.VllmConfig`.
|
|
||||||
"""
|
|
||||||
from vllm.entrypoints.chat_utils import load_chat_template
|
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
|
||||||
from vllm.renderers import renderer_from_config
|
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
|
||||||
|
|
||||||
served_model_names = args.served_model_name or [args.model]
|
|
||||||
model_registry = OpenAIModelRegistry(
|
|
||||||
model_config=vllm_config.model_config,
|
|
||||||
base_model_paths=[
|
|
||||||
BaseModelPath(name=name, model_path=args.model)
|
|
||||||
for name in served_model_names
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.enable_log_requests:
|
|
||||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
|
||||||
else:
|
|
||||||
request_logger = None
|
|
||||||
|
|
||||||
renderer = renderer_from_config(vllm_config)
|
|
||||||
resolved_chat_template = load_chat_template(args.chat_template)
|
|
||||||
|
|
||||||
state.online_renderer = OnlineRenderer(
|
|
||||||
model_config=vllm_config.model_config,
|
|
||||||
renderer=renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
|
||||||
tool_parser=args.tool_call_parser,
|
|
||||||
reasoning_parser=args.reasoning_parser,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
log_error_stack=args.log_error_stack,
|
|
||||||
)
|
|
||||||
state.online_renderer.warmup()
|
|
||||||
|
|
||||||
state.online_derenderer = OnlineDerenderer(
|
|
||||||
model_config=vllm_config.model_config,
|
|
||||||
renderer=renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
|
||||||
tool_parser=args.tool_call_parser,
|
|
||||||
reasoning_parser=args.reasoning_parser,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
log_error_stack=args.log_error_stack,
|
|
||||||
)
|
|
||||||
|
|
||||||
state.openai_serving_models = model_registry
|
|
||||||
state.serving_tokenization = ServingTokenization(
|
|
||||||
model_registry,
|
|
||||||
state.online_renderer,
|
|
||||||
request_logger=request_logger,
|
|
||||||
chat_template=resolved_chat_template,
|
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
|
||||||
)
|
|
||||||
|
|
||||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
|
||||||
|
|
||||||
init_render_state(state, request_logger)
|
|
||||||
|
|
||||||
state.vllm_config = vllm_config
|
|
||||||
# Disable stats logging — there is no engine to poll.
|
|
||||||
state.log_stats = False
|
|
||||||
state.engine_client = None
|
|
||||||
state.args = args
|
|
||||||
state.enable_server_load_tracking = False
|
|
||||||
state.server_load_metrics = 0
|
|
||||||
|
|
||||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
|
||||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
|
||||||
await _init_endpoint_plugins_state(None, state, args)
|
|
||||||
|
|
||||||
|
|
||||||
def create_server_socket(
|
|
||||||
addr: tuple[str, int],
|
|
||||||
*,
|
|
||||||
reuse_port: bool,
|
|
||||||
) -> socket.socket:
|
|
||||||
family = socket.AF_INET
|
|
||||||
if is_valid_ipv6_address(addr[0]):
|
|
||||||
family = socket.AF_INET6
|
|
||||||
|
|
||||||
sock = socket.socket(family=family, type=socket.SOCK_STREAM)
|
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
||||||
if reuse_port:
|
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
|
||||||
sock.bind(addr)
|
|
||||||
|
|
||||||
return sock
|
|
||||||
|
|
||||||
|
|
||||||
def create_server_unix_socket(path: str) -> socket.socket:
|
|
||||||
sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
|
|
||||||
sock.bind(path)
|
|
||||||
return sock
|
|
||||||
|
|
||||||
|
|
||||||
def validate_api_server_args(args):
|
|
||||||
valid_tool_parses = ToolParserManager.list_registered()
|
|
||||||
if args.enable_auto_tool_choice and args.tool_call_parser not in valid_tool_parses:
|
|
||||||
raise KeyError(
|
|
||||||
f"invalid tool call parser: {args.tool_call_parser} "
|
|
||||||
f"(chose from {{ {','.join(valid_tool_parses)} }})"
|
|
||||||
)
|
|
||||||
|
|
||||||
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
|
||||||
if (
|
|
||||||
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
|
||||||
) and reasoning_parser not in valid_reasoning_parsers:
|
|
||||||
raise KeyError(
|
|
||||||
f"invalid reasoning parser: {reasoning_parser} "
|
|
||||||
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@instrument(span_name="API server setup")
|
|
||||||
def setup_server(args, *, reuse_port: bool):
|
|
||||||
"""Validate API server args and create the server socket."""
|
|
||||||
|
|
||||||
log_version_and_model(logger, VLLM_VERSION, args.model)
|
|
||||||
log_non_default_args(args)
|
|
||||||
|
|
||||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
|
||||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
|
||||||
|
|
||||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
|
||||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
|
||||||
|
|
||||||
validate_api_server_args(args)
|
|
||||||
|
|
||||||
# workaround to make sure that we bind the port before the engine is set up.
|
|
||||||
# This avoids race conditions with ray.
|
|
||||||
# see https://github.com/vllm-project/vllm/issues/8204
|
|
||||||
if args.uds:
|
|
||||||
sock = create_server_unix_socket(args.uds)
|
|
||||||
else:
|
|
||||||
sock_addr = (args.host or "", args.port)
|
|
||||||
sock = create_server_socket(sock_addr, reuse_port=reuse_port)
|
|
||||||
|
|
||||||
# workaround to avoid footguns where uvicorn drops requests with too
|
|
||||||
# many concurrent requests active
|
|
||||||
set_ulimit()
|
|
||||||
|
|
||||||
if args.uds:
|
|
||||||
listen_address = f"unix:{args.uds}"
|
|
||||||
else:
|
|
||||||
addr, port = sock_addr
|
|
||||||
is_ssl = args.ssl_keyfile and args.ssl_certfile
|
|
||||||
host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
|
|
||||||
listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
|
|
||||||
return listen_address, sock
|
|
||||||
|
|
||||||
|
|
||||||
async def build_and_serve(
|
|
||||||
engine_client: EngineClient,
|
|
||||||
listen_address: str,
|
|
||||||
sock: socket.socket,
|
|
||||||
args: Namespace,
|
|
||||||
**uvicorn_kwargs,
|
|
||||||
) -> asyncio.Task:
|
|
||||||
"""Build FastAPI app, initialize state, and start serving.
|
|
||||||
|
|
||||||
Returns the shutdown task for the caller to await.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Get uvicorn log config (from file or with endpoint filter)
|
|
||||||
log_config = get_uvicorn_log_config(args)
|
|
||||||
if log_config is not None:
|
|
||||||
uvicorn_kwargs["log_config"] = log_config
|
|
||||||
|
|
||||||
supported_tasks = await engine_client.get_supported_tasks()
|
|
||||||
model_config = engine_client.model_config
|
|
||||||
|
|
||||||
logger.info("Supported tasks: %s", supported_tasks)
|
|
||||||
app = build_app(args, supported_tasks, model_config)
|
|
||||||
await init_app_state(engine_client, app.state, args, supported_tasks)
|
|
||||||
|
|
||||||
logger.info("Starting vLLM server on %s", listen_address)
|
|
||||||
|
|
||||||
return await serve_http(
|
|
||||||
app,
|
|
||||||
sock=sock,
|
|
||||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
log_level=args.uvicorn_log_level,
|
|
||||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
|
||||||
# no access log will be output.
|
|
||||||
access_log=not args.disable_uvicorn_access_log,
|
|
||||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
|
||||||
ssl_keyfile=args.ssl_keyfile,
|
|
||||||
ssl_certfile=args.ssl_certfile,
|
|
||||||
ssl_ca_certs=args.ssl_ca_certs,
|
|
||||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
|
||||||
ssl_ciphers=args.ssl_ciphers,
|
|
||||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
|
||||||
h11_max_header_count=args.h11_max_header_count,
|
|
||||||
**uvicorn_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def build_and_serve_renderer(
|
|
||||||
vllm_config: VllmConfig,
|
|
||||||
listen_address: str,
|
|
||||||
sock: socket.socket,
|
|
||||||
args: Namespace,
|
|
||||||
**uvicorn_kwargs,
|
|
||||||
) -> asyncio.Task:
|
|
||||||
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
|
||||||
start serving.
|
|
||||||
|
|
||||||
Returns the shutdown task for the caller to await.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Get uvicorn log config (from file or with endpoint filter)
|
|
||||||
log_config = get_uvicorn_log_config(args)
|
|
||||||
if log_config is not None:
|
|
||||||
uvicorn_kwargs["log_config"] = log_config
|
|
||||||
|
|
||||||
app = build_app(args, ("render",))
|
|
||||||
await init_render_app_state(vllm_config, app.state, args)
|
|
||||||
|
|
||||||
logger.info("Starting vLLM server on %s", listen_address)
|
|
||||||
|
|
||||||
return await serve_http(
|
|
||||||
app,
|
|
||||||
sock=sock,
|
|
||||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
log_level=args.uvicorn_log_level,
|
|
||||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
|
||||||
# no access log will be output.
|
|
||||||
access_log=not args.disable_uvicorn_access_log,
|
|
||||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
|
||||||
ssl_keyfile=args.ssl_keyfile,
|
|
||||||
ssl_certfile=args.ssl_certfile,
|
|
||||||
ssl_ca_certs=args.ssl_ca_certs,
|
|
||||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
|
||||||
ssl_ciphers=args.ssl_ciphers,
|
|
||||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
|
||||||
h11_max_header_count=args.h11_max_header_count,
|
|
||||||
**uvicorn_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
|
||||||
"""Run a single-worker API server."""
|
|
||||||
|
|
||||||
decorate_logs("APIServer", skip_if_decorated=True)
|
|
||||||
|
|
||||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
|
||||||
# own signal handlers. Once uvicorn is running it replaces this.
|
|
||||||
def _interrupt_init(*_) -> None:
|
|
||||||
raise KeyboardInterrupt("terminated")
|
|
||||||
|
|
||||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
|
||||||
|
|
||||||
listen_address, sock = setup_server(args, reuse_port=False)
|
|
||||||
await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
async def run_server_worker(
|
|
||||||
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
|
||||||
) -> None:
|
|
||||||
"""Run a single API server worker."""
|
|
||||||
|
|
||||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
|
||||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
|
||||||
|
|
||||||
if args.reasoning_parser_plugin and len(args.reasoning_parser_plugin) > 3:
|
|
||||||
ReasoningParserManager.import_reasoning_parser(args.reasoning_parser_plugin)
|
|
||||||
|
|
||||||
async with build_async_engine_client(
|
|
||||||
args,
|
|
||||||
client_config=client_config,
|
|
||||||
) as engine_client:
|
|
||||||
shutdown_task = await build_and_serve(
|
|
||||||
engine_client, listen_address, sock, args, **uvicorn_kwargs
|
|
||||||
)
|
|
||||||
# NB: Await server shutdown only after the backend context is exited
|
|
||||||
try:
|
|
||||||
await shutdown_task
|
|
||||||
finally:
|
|
||||||
sock.close()
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"build_async_engine_client",
|
||||||
|
"build_async_engine_client_from_engine_args",
|
||||||
|
"build_app",
|
||||||
|
"init_app_state",
|
||||||
|
"init_render_app_state",
|
||||||
|
"create_server_socket",
|
||||||
|
"create_server_unix_socket",
|
||||||
|
"validate_api_server_args",
|
||||||
|
"setup_server",
|
||||||
|
"build_and_serve",
|
||||||
|
"build_and_serve_renderer",
|
||||||
|
"run_server",
|
||||||
|
"run_server_worker",
|
||||||
|
"register_api_routers",
|
||||||
|
]
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# NOTE(simon):
|
warnings.warn(
|
||||||
# This section should be in sync with vllm/entrypoints/cli/main.py for CLI
|
"The `python -m vllm.entrypoints.openai.api_server` command is deprecated "
|
||||||
# entrypoints.
|
"and may be removed in a future release. Please use `vllm server` instead.",
|
||||||
cli_env_setup()
|
DeprecationWarning,
|
||||||
parser = FlexibleArgumentParser(
|
stacklevel=1,
|
||||||
description="vLLM OpenAI-Compatible RESTful API server."
|
|
||||||
)
|
)
|
||||||
parser = make_arg_parser(parser)
|
from vllm.entrypoints.launchers.api_server.entry import main
|
||||||
args = parser.parse_args()
|
|
||||||
validate_parsed_serve_args(args)
|
|
||||||
|
|
||||||
uvloop.run(run_server(args))
|
main()
|
||||||
|
|||||||
@@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
validate_chat_template,
|
validate_chat_template,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
from vllm.entrypoints.launchers.utils.constants import (
|
||||||
from vllm.entrypoints.serve.utils.constants import (
|
|
||||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||||
from vllm.tool_parsers import ToolParserManager
|
from vllm.tool_parsers import ToolParserManager
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
@@ -444,6 +444,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
|||||||
|
|
||||||
def create_parser_for_docs() -> FlexibleArgumentParser:
|
def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||||
parser_for_docs = FlexibleArgumentParser(
|
parser_for_docs = FlexibleArgumentParser(
|
||||||
prog="-m vllm.entrypoints.openai.api_server"
|
prog="-m vllm.entrypoints.launchers.api_server.entry"
|
||||||
)
|
)
|
||||||
return make_arg_parser(parser_for_docs)
|
return make_arg_parser(parser_for_docs)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import uvloop
|
|||||||
from fastapi import FastAPI, Response
|
from fastapi import FastAPI, Response
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.launcher import NoSignalServer
|
from vllm.entrypoints.launchers.launcher import NoSignalServer
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils.system_utils import (
|
from vllm.utils.system_utils import (
|
||||||
decorate_logs,
|
decorate_logs,
|
||||||
@@ -236,7 +236,7 @@ def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
|
|||||||
|
|
||||||
|
|
||||||
def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||||
from vllm.entrypoints.openai.api_server import run_server
|
from vllm.entrypoints.launchers.api_server.entry import run_server
|
||||||
|
|
||||||
uvloop.run(run_server(child_args))
|
uvloop.run(run_server(child_args))
|
||||||
|
|
||||||
|
|||||||
@@ -39,7 +39,6 @@ from vllm.entrypoints.openai.responses.utils import (
|
|||||||
build_response_output_items,
|
build_response_output_items,
|
||||||
construct_tool_dicts,
|
construct_tool_dicts,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
|
||||||
from vllm.outputs import RequestOutput
|
from vllm.outputs import RequestOutput
|
||||||
from vllm.parser.abstract_parser import Parser
|
from vllm.parser.abstract_parser import Parser
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
@@ -58,6 +57,7 @@ _TOOL_NAME_TO_TYPE_MAP = {
|
|||||||
"python": "code_interpreter",
|
"python": "code_interpreter",
|
||||||
"container": "container",
|
"container": "container",
|
||||||
}
|
}
|
||||||
|
MCP_PREFIX = "mcp_"
|
||||||
|
|
||||||
|
|
||||||
def _map_tool_name_to_tool_type(tool_name: str) -> str:
|
def _map_tool_name_to_tool_type(tool_name: str) -> str:
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ from vllm.config import config
|
|||||||
from vllm.connections import global_http_connection
|
from vllm.connections import global_http_connection
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.openai.api_server import init_app_state
|
from vllm.entrypoints.launchers.api_server.entry import init_app_state
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
@@ -836,8 +836,7 @@ async def run_batch(
|
|||||||
error_msg=f"URL {request.url} was used. "
|
error_msg=f"URL {request.url} was used. "
|
||||||
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
|
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
|
||||||
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
|
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
|
||||||
" /rerank. See vllm/entrypoints/openai/api_server.py "
|
" /rerank.",
|
||||||
"for supported score/rerank versions.",
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -848,7 +847,7 @@ async def run_batch(
|
|||||||
|
|
||||||
|
|
||||||
async def main(args: Namespace):
|
async def main(args: Namespace):
|
||||||
from vllm.entrypoints.openai.api_server import build_async_engine_client
|
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
|
||||||
from vllm.usage.usage_lib import UsageContext
|
from vllm.usage.usage_lib import UsageContext
|
||||||
|
|
||||||
validate_run_batch_args(args)
|
validate_run_batch_args(args)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
from vllm.entrypoints.launcher import terminate_if_errored
|
from vllm.entrypoints.launchers.launcher import terminate_if_errored
|
||||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||||
from vllm.exceptions import VLLMError
|
from vllm.exceptions import VLLMError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ for the loading/gating rules and `docs/usage/security.md` for the security
|
|||||||
posture of exposing plugin defined routes.
|
posture of exposing plugin defined routes.
|
||||||
|
|
||||||
The CPU only render server (see `build_and_serve_renderer` in
|
The CPU only render server (see `build_and_serve_renderer` in
|
||||||
`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin
|
`vllm/entrypoints/launchers/render`) has no `EngineClient`. A plugin
|
||||||
eligible for the `render` task (`required_tasks` is `None` or includes
|
eligible for the `render` task (`required_tasks` is `None` or includes
|
||||||
`"render"`) still gets `attach_router` called but `init_state` receives
|
`"render"`) still gets `attach_router` called but `init_state` receives
|
||||||
`engine_client=None`. Plugins that cannot function without an engine should
|
`engine_client=None`. Plugins that cannot function without an engine should
|
||||||
@@ -34,8 +34,9 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from starlette.datastructures import State
|
from starlette.datastructures import State
|
||||||
|
|
||||||
|
from vllm.engine.protocol import EngineClient
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.engine.protocol import EngineClient
|
|
||||||
from vllm.tasks import SupportedTask
|
from vllm.tasks import SupportedTask
|
||||||
|
|
||||||
|
|
||||||
@@ -85,3 +86,38 @@ class EndpointPlugin(Protocol):
|
|||||||
cannot function without an engine.
|
cannot function without an engine.
|
||||||
"""
|
"""
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
def attach_endpoint_plugins(
|
||||||
|
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||||
|
) -> None:
|
||||||
|
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
|
||||||
|
|
||||||
|
Attached last after all core routers. This is so endpoint plugin routes can
|
||||||
|
shadow core routes with the same path (see `EndpointPlugin.attach_router`
|
||||||
|
docstring). No-ops when no plugins are discovered/allowlisted.
|
||||||
|
"""
|
||||||
|
from vllm.plugins import load_endpoint_plugins
|
||||||
|
|
||||||
|
endpoint_plugins = load_endpoint_plugins(supported_tasks)
|
||||||
|
for plugin in endpoint_plugins:
|
||||||
|
plugin.attach_router(app)
|
||||||
|
app.state.endpoint_plugins = endpoint_plugins
|
||||||
|
|
||||||
|
|
||||||
|
async def init_endpoint_plugins_state(
|
||||||
|
engine_client: EngineClient | None, state: State, args: Namespace
|
||||||
|
) -> None:
|
||||||
|
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
|
||||||
|
|
||||||
|
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
|
||||||
|
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
|
||||||
|
build their own bare `State` without going through `build_app`. As a result
|
||||||
|
`endpoint_plugins` may be absent and are treated that the same as "none attached".
|
||||||
|
|
||||||
|
`engine_client` is `None` for the CPU only render server which has no
|
||||||
|
engine (see `init_render_app_state`). Plugins must handle a `None`
|
||||||
|
`engine_client` themselves (see `EndpointPlugin.init_state`).
|
||||||
|
"""
|
||||||
|
for plugin in getattr(state, "endpoint_plugins", []):
|
||||||
|
await plugin.init_state(engine_client, state, args)
|
||||||
|
|||||||
@@ -41,3 +41,4 @@ FrontendTask = Literal["render"]
|
|||||||
FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)
|
FRONTEND_TASKS: tuple[FrontendTask, ...] = get_args(FrontendTask)
|
||||||
|
|
||||||
SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]
|
SupportedTask = Literal[GenerationTask, PoolingTask, FrontendTask]
|
||||||
|
FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
|
||||||
|
|||||||
+1
-1
@@ -503,7 +503,7 @@ def run_api_server_worker_proc(
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""Entrypoint for individual API server worker processes."""
|
"""Entrypoint for individual API server worker processes."""
|
||||||
|
|
||||||
from vllm.entrypoints.openai.api_server import run_server_worker
|
from vllm.entrypoints.launchers.api_server.entry import run_server_worker
|
||||||
|
|
||||||
client_config = client_config or {}
|
client_config = client_config or {}
|
||||||
server_index = client_config.get("client_index", 0)
|
server_index = client_config.get("client_index", 0)
|
||||||
|
|||||||
Reference in New Issue
Block a user