[Frontend] Consolidate online serving utils. (#44479)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,348 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import os
|
||||
from argparse import Namespace
|
||||
from logging import Logger
|
||||
from string import Template
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask, BackgroundTasks
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import current_formatter_type, init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
VLLM_SUBCMD_PARSER_EPILOG = (
|
||||
"For full list: vllm {subcmd} --help=all\n"
|
||||
"For a section: vllm {subcmd} --help=ModelConfig (case-insensitive)\n" # noqa: E501
|
||||
"For a flag: vllm {subcmd} --help=max-model-len (_ or - accepted)\n" # noqa: E501
|
||||
"Documentation: https://docs.vllm.ai\n"
|
||||
)
|
||||
|
||||
|
||||
async def listen_for_disconnect(request: Request) -> None:
|
||||
"""Returns if a disconnect message is received"""
|
||||
while True:
|
||||
message = await request.receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
# If load tracking is enabled *and* the counter exists, decrement
|
||||
# it. Combines the previous nested checks into a single condition
|
||||
# to satisfy the linter rule.
|
||||
if getattr(
|
||||
request.app.state, "enable_server_load_tracking", False
|
||||
) and hasattr(request.app.state, "server_load_metrics"):
|
||||
request.app.state.server_load_metrics -= 1
|
||||
break
|
||||
|
||||
|
||||
def with_cancellation(handler_func):
|
||||
"""Decorator that allows a route handler to be cancelled by client
|
||||
disconnections.
|
||||
|
||||
This does _not_ use request.is_disconnected, which does not work with
|
||||
middleware. Instead this follows the pattern from
|
||||
starlette.StreamingResponse, which simultaneously awaits on two tasks- one
|
||||
to wait for an http disconnect message, and the other to do the work that we
|
||||
want done. When the first task finishes, the other is cancelled.
|
||||
|
||||
A core assumption of this method is that the body of the request has already
|
||||
been read. This is a safe assumption to make for fastapi handlers that have
|
||||
already parsed the body of the request into a pydantic model for us.
|
||||
This decorator is unsafe to use elsewhere, as it will consume and throw away
|
||||
all incoming messages for the request while it looks for a disconnect
|
||||
message.
|
||||
|
||||
In the case where a `StreamingResponse` is returned by the handler, this
|
||||
wrapper will stop listening for disconnects and instead the response object
|
||||
will start listening for disconnects.
|
||||
"""
|
||||
|
||||
# Functools.wraps is required for this wrapper to appear to fastapi as a
|
||||
# normal route handler, with the correct request type hinting.
|
||||
@functools.wraps(handler_func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
# The request is either the second positional arg or `raw_request`
|
||||
request = args[1] if len(args) > 1 else kwargs["raw_request"]
|
||||
|
||||
handler_task = asyncio.create_task(handler_func(*args, **kwargs))
|
||||
cancellation_task = asyncio.create_task(listen_for_disconnect(request))
|
||||
|
||||
done, pending = await asyncio.wait(
|
||||
[handler_task, cancellation_task], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
|
||||
if handler_task in done:
|
||||
return handler_task.result()
|
||||
return None
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def decrement_server_load(request: Request):
|
||||
request.app.state.server_load_metrics -= 1
|
||||
|
||||
|
||||
def load_aware_call(func):
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
raw_request = kwargs.get("raw_request", args[1] if len(args) > 1 else None)
|
||||
|
||||
if raw_request is None:
|
||||
raise ValueError(
|
||||
"raw_request required when server load tracking is enabled"
|
||||
)
|
||||
|
||||
if not getattr(raw_request.app.state, "enable_server_load_tracking", False):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
# ensure the counter exists
|
||||
if not hasattr(raw_request.app.state, "server_load_metrics"):
|
||||
raw_request.app.state.server_load_metrics = 0
|
||||
|
||||
raw_request.app.state.server_load_metrics += 1
|
||||
try:
|
||||
response = await func(*args, **kwargs)
|
||||
except Exception:
|
||||
raw_request.app.state.server_load_metrics -= 1
|
||||
raise
|
||||
|
||||
if isinstance(response, (JSONResponse, StreamingResponse)):
|
||||
if response.background is None:
|
||||
response.background = BackgroundTask(decrement_server_load, raw_request)
|
||||
elif isinstance(response.background, BackgroundTasks):
|
||||
response.background.add_task(decrement_server_load, raw_request)
|
||||
elif isinstance(response.background, BackgroundTask):
|
||||
# Convert the single BackgroundTask to BackgroundTasks
|
||||
# and chain the decrement_server_load task to it
|
||||
tasks = BackgroundTasks()
|
||||
tasks.add_task(
|
||||
response.background.func,
|
||||
*response.background.args,
|
||||
**response.background.kwargs,
|
||||
)
|
||||
tasks.add_task(decrement_server_load, raw_request)
|
||||
response.background = tasks
|
||||
else:
|
||||
raw_request.app.state.server_load_metrics -= 1
|
||||
|
||||
return response
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def cli_env_setup():
|
||||
# The safest multiprocessing method is `spawn`, as the default `fork` method
|
||||
# is not compatible with some accelerators. The default method will be
|
||||
# changing in future versions of Python, so we should use it explicitly when
|
||||
# possible.
|
||||
#
|
||||
# We only set it here in the CLI entrypoint, because changing to `spawn`
|
||||
# could break some existing code using vLLM as a library. `spawn` will cause
|
||||
# unexpected behavior if the code is not protected by
|
||||
# `if __name__ == "__main__":`.
|
||||
#
|
||||
# References:
|
||||
# - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
|
||||
# - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
|
||||
# - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
|
||||
# - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
|
||||
if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
|
||||
logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
def get_max_tokens(
|
||||
max_model_len: int,
|
||||
max_tokens: int | None,
|
||||
input_length: int,
|
||||
default_sampling_params: dict,
|
||||
override_max_tokens: int | None = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
) -> int:
|
||||
if truncate_prompt_tokens is not None:
|
||||
limit = truncate_prompt_tokens
|
||||
input_length = min(
|
||||
input_length,
|
||||
max_model_len if limit == -1 else limit,
|
||||
)
|
||||
if max_model_len < input_length:
|
||||
raise ValueError(
|
||||
f"Input length ({input_length}) exceeds model's maximum "
|
||||
f"context length ({max_model_len})."
|
||||
)
|
||||
model_max_tokens = max_model_len - input_length
|
||||
platform_max_tokens = current_platform.get_max_output_tokens(input_length)
|
||||
fallback_max_tokens = (
|
||||
max_tokens
|
||||
if max_tokens is not None
|
||||
else default_sampling_params.get("max_tokens")
|
||||
)
|
||||
|
||||
return min(
|
||||
val
|
||||
for val in (
|
||||
model_max_tokens,
|
||||
fallback_max_tokens,
|
||||
override_max_tokens,
|
||||
platform_max_tokens,
|
||||
)
|
||||
if val is not None
|
||||
)
|
||||
|
||||
|
||||
def get_non_default_args(args: Namespace | EngineArgs) -> dict[str, Any]:
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
|
||||
non_default_args = {}
|
||||
|
||||
# Handle Namespace
|
||||
if isinstance(args, Namespace):
|
||||
parser = make_arg_parser(FlexibleArgumentParser())
|
||||
for arg, default in vars(parser.parse_args([])).items():
|
||||
if default != getattr(args, arg):
|
||||
non_default_args[arg] = getattr(args, arg)
|
||||
|
||||
# Handle EngineArgs instance
|
||||
elif isinstance(args, EngineArgs):
|
||||
default_args = EngineArgs(model=args.model) # Create default instance
|
||||
for field in dataclasses.fields(args):
|
||||
current_val = getattr(args, field.name)
|
||||
default_val = getattr(default_args, field.name)
|
||||
if current_val != default_val:
|
||||
non_default_args[field.name] = current_val
|
||||
if default_args.model != EngineArgs.model:
|
||||
non_default_args["model"] = default_args.model
|
||||
else:
|
||||
raise TypeError(
|
||||
"Unsupported argument type. Must be Namespace or EngineArgs instance."
|
||||
)
|
||||
|
||||
return non_default_args
|
||||
|
||||
|
||||
def _jsonify_arg_value(value: Any) -> Any:
|
||||
if value is None or isinstance(value, bool | int | float | str):
|
||||
return value
|
||||
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
||||
return {
|
||||
key: _jsonify_arg_value(val)
|
||||
for key, val in dataclasses.asdict(value).items()
|
||||
}
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _jsonify_arg_value(val) for key, val in value.items()}
|
||||
if isinstance(value, tuple | list):
|
||||
return [_jsonify_arg_value(item) for item in value]
|
||||
if (model_dump := getattr(value, "model_dump", None)) is not None:
|
||||
return _jsonify_arg_value(model_dump(mode="json"))
|
||||
if (to_dict := getattr(value, "dict", None)) is not None:
|
||||
return _jsonify_arg_value(to_dict())
|
||||
return repr(value)
|
||||
|
||||
|
||||
def jsonify_non_default_args(
|
||||
args: Namespace | EngineArgs,
|
||||
*,
|
||||
exclude: set[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
non_default_args = get_non_default_args(args)
|
||||
if exclude is not None:
|
||||
for key in exclude:
|
||||
non_default_args.pop(key, None)
|
||||
|
||||
return {key: _jsonify_arg_value(value) for key, value in non_default_args.items()}
|
||||
|
||||
|
||||
def log_non_default_args(args: Namespace | EngineArgs):
|
||||
non_default_args = get_non_default_args(args)
|
||||
logger.info("non-default args: %s", non_default_args)
|
||||
|
||||
|
||||
def should_include_usage(
|
||||
stream_options: StreamOptions | None, enable_force_include_usage: bool
|
||||
) -> tuple[bool, bool]:
|
||||
if enable_force_include_usage:
|
||||
return True, True
|
||||
if stream_options:
|
||||
include_usage = bool(stream_options.include_usage)
|
||||
include_continuous_usage = include_usage and bool(
|
||||
stream_options.continuous_usage_stats
|
||||
)
|
||||
else:
|
||||
include_usage, include_continuous_usage = False, False
|
||||
return include_usage, include_continuous_usage
|
||||
|
||||
|
||||
def process_lora_modules(
|
||||
args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
|
||||
) -> list[LoRAModulePath]:
|
||||
from vllm.entrypoints.openai.models.serving import LoRAModulePath
|
||||
|
||||
lora_modules = args_lora_modules
|
||||
if default_mm_loras:
|
||||
default_mm_lora_paths = [
|
||||
LoRAModulePath(
|
||||
name=modality,
|
||||
path=lora_path,
|
||||
)
|
||||
for modality, lora_path in default_mm_loras.items()
|
||||
]
|
||||
if args_lora_modules is None:
|
||||
lora_modules = default_mm_lora_paths
|
||||
else:
|
||||
lora_modules += default_mm_lora_paths
|
||||
return lora_modules
|
||||
|
||||
|
||||
def sanitize_message(message: str) -> str:
|
||||
# Avoid leaking memory address from object reprs
|
||||
return re.sub(r" at 0x[0-9a-f]+>", ">", message)
|
||||
|
||||
|
||||
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
|
||||
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
|
||||
message = "vLLM server version %s, serving model %s"
|
||||
else:
|
||||
logo_template = Template(
|
||||
"\n ${w}█ █ █▄ ▄█${r}\n"
|
||||
" ${o}▄▄${r} ${b}▄█${r} ${w}█ █ █ ▀▄▀ █${r} version ${w}%s${r}\n"
|
||||
" ${o}█${r}${b}▄█▀${r} ${w}█ █ █ █${r} model ${w}%s${r}\n"
|
||||
" ${b}▀▀${r} ${w}▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀${r}\n"
|
||||
)
|
||||
colors = {
|
||||
"w": "\033[97;1m", # white
|
||||
"o": "\033[93m", # orange
|
||||
"b": "\033[94m", # blue
|
||||
"r": "\033[0m", # reset
|
||||
}
|
||||
if formatter != "color":
|
||||
# monochrome logo (no ansi escape codes)
|
||||
colors = dict.fromkeys(colors, "")
|
||||
|
||||
message = logo_template.substitute(colors)
|
||||
|
||||
lgr.info(message, version, model_name)
|
||||
|
||||
|
||||
async def validate_json_request(raw_request: Request):
|
||||
content_type = raw_request.headers.get("content-type", "").lower()
|
||||
media_type = content_type.split(";", maxsplit=1)[0]
|
||||
if media_type != "application/json":
|
||||
raise RequestValidationError(
|
||||
errors=["Unsupported Media Type: Only 'application/json' is allowed"]
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared constants for vLLM entrypoints.
|
||||
"""
|
||||
|
||||
# HTTP header limits for h11 parser
|
||||
# These constants help mitigate header abuse attacks
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
|
||||
H11_MAX_HEADER_COUNT_DEFAULT = 256
|
||||
|
||||
MCP_PREFIX = "mcp_"
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def create_error_response(
|
||||
message: str | Exception,
|
||||
err_type: str = "BadRequestError",
|
||||
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
|
||||
param: str | None = None,
|
||||
) -> ErrorResponse:
|
||||
exc: Exception | None = None
|
||||
|
||||
if isinstance(message, Exception):
|
||||
exc = message
|
||||
logger.debug(
|
||||
"create_error_response called with %s: %s", type(exc).__name__, exc
|
||||
)
|
||||
|
||||
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
|
||||
|
||||
if isinstance(exc, VLLMValidationError):
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = exc.parameter
|
||||
elif isinstance(exc, VLLMNotFoundError):
|
||||
err_type = "NotFoundError"
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
param = None
|
||||
elif isinstance(exc, (ValueError, TypeError, OverflowError)):
|
||||
# Common validation errors from user input
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = None
|
||||
elif isinstance(exc, NotImplementedError):
|
||||
err_type = "NotImplementedError"
|
||||
status_code = HTTPStatus.NOT_IMPLEMENTED
|
||||
param = None
|
||||
elif isinstance(exc, GenerationError):
|
||||
err_type = "InternalServerError"
|
||||
status_code = exc.status_code
|
||||
param = None
|
||||
elif any(cls.__name__ == "TemplateError" for cls in type(exc).__mro__):
|
||||
# jinja2.TemplateError and its subclasses (avoid importing jinja2)
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = None
|
||||
else:
|
||||
err_type = "InternalServerError"
|
||||
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
param = None
|
||||
|
||||
message = str(exc)
|
||||
|
||||
return ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=err_type,
|
||||
code=status_code.value,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Build the ``system_fingerprint`` string returned by the OpenAI-compatible
|
||||
server.
|
||||
|
||||
Four modes, configured via ``--fingerprint-mode``:
|
||||
|
||||
* ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>`` — encodes
|
||||
server version, any non-trivial parallelism degree (tp/pp/dp/ep), and an
|
||||
8-char prefix of ``vllm_config.compute_hash()`` (covers model identity,
|
||||
quant config, speculative, attention backend, etc.).
|
||||
* ``hash``: ``vllm-<version>-<hash8>`` — parallelism stripped.
|
||||
* ``custom``: user-provided literal via ``--fingerprint-value``.
|
||||
* ``none``: the field is omitted (serialized as ``null``).
|
||||
|
||||
``get_system_fingerprint`` is only called at serving-class init (a handful
|
||||
of times per server); each subclass caches the returned string on
|
||||
``self.system_fingerprint``, so per-request cost is one attribute read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
FingerprintMode = Literal["full", "hash", "custom", "none"]
|
||||
|
||||
_DEFAULT_MODE: FingerprintMode = "full"
|
||||
_CUSTOM_VALUE: str | None = None
|
||||
|
||||
|
||||
def set_default_fingerprint_mode(
|
||||
mode: FingerprintMode,
|
||||
custom_value: str | None = None,
|
||||
) -> None:
|
||||
"""Configure the fingerprint mode for subsequent ``get_system_fingerprint``
|
||||
calls. Called once at server startup."""
|
||||
global _DEFAULT_MODE, _CUSTOM_VALUE
|
||||
_DEFAULT_MODE = mode
|
||||
_CUSTOM_VALUE = custom_value
|
||||
|
||||
|
||||
def get_system_fingerprint(vllm_config: Any) -> str | None:
|
||||
"""Return the fingerprint for ``vllm_config`` using the mode configured by
|
||||
``set_default_fingerprint_mode``."""
|
||||
return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
|
||||
|
||||
|
||||
def build_system_fingerprint(
|
||||
vllm_config: Any,
|
||||
mode: FingerprintMode = "full",
|
||||
custom_value: str | None = None,
|
||||
) -> str | None:
|
||||
if mode == "none":
|
||||
return None
|
||||
if mode == "custom":
|
||||
return custom_value
|
||||
|
||||
from vllm import __version__ as vllm_version
|
||||
|
||||
try:
|
||||
hash8 = vllm_config.compute_hash()[:8]
|
||||
except Exception:
|
||||
hash8 = "nohash"
|
||||
|
||||
if mode == "hash":
|
||||
return f"vllm-{vllm_version}-{hash8}"
|
||||
|
||||
# mode == "full"
|
||||
parts: list[str] = [f"vllm-{vllm_version}"]
|
||||
pc = getattr(vllm_config, "parallel_config", None)
|
||||
if pc is not None:
|
||||
tp = getattr(pc, "tensor_parallel_size", 1)
|
||||
if tp > 1:
|
||||
parts.append(f"tp{tp}")
|
||||
pp = getattr(pc, "pipeline_parallel_size", 1)
|
||||
if pp > 1:
|
||||
parts.append(f"pp{pp}")
|
||||
dp = getattr(pc, "data_parallel_size", 1)
|
||||
if dp > 1:
|
||||
parts.append(f"dp{dp}")
|
||||
if getattr(pc, "enable_expert_parallel", False):
|
||||
parts.append("ep")
|
||||
parts.append(hash8)
|
||||
return "-".join(parts)
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utility functions that create ORCA endpoint load report response headers.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.metrics.reader import Gauge, get_metrics_snapshot
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def create_orca_header(
|
||||
metrics_format: str, named_metrics: list[tuple[str, float]]
|
||||
) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format
|
||||
and adds custom metrics to named_metrics.
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
- named_metrics (List[Tuple[str, float]]): List of tuples with metric names
|
||||
and their corresponding double values.
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
"""
|
||||
|
||||
if metrics_format.lower() not in ["text", "json"]:
|
||||
logger.warning(
|
||||
"Warning: `%s` format is not supported in the ORCA response header",
|
||||
format,
|
||||
)
|
||||
return None
|
||||
|
||||
header = {}
|
||||
orca_report = {
|
||||
"named_metrics": {
|
||||
metric_name: value
|
||||
for metric_name, value in named_metrics
|
||||
if isinstance(metric_name, str) and isinstance(value, float)
|
||||
}
|
||||
}
|
||||
# output example:
|
||||
# endpoint-load-metrics: TEXT named_metrics.kv_cache_utilization=0.4
|
||||
if metrics_format.lower() == "text":
|
||||
native_http_header = ", ".join(
|
||||
[
|
||||
f"named_metrics.{metric_name}={value}"
|
||||
for metric_name, value in named_metrics
|
||||
if isinstance(metric_name, str) and isinstance(value, float)
|
||||
]
|
||||
)
|
||||
header["endpoint-load-metrics"] = f"TEXT {native_http_header}"
|
||||
|
||||
# output example:
|
||||
# endpoint-load-metrics: JSON “named_metrics”: {“custom-metric-util”: 0.4}
|
||||
elif metrics_format.lower() == "json":
|
||||
header["endpoint-load-metrics"] = f"JSON {json.dumps(orca_report)}"
|
||||
|
||||
logger.info("Created ORCA header %s", header)
|
||||
|
||||
return header
|
||||
|
||||
|
||||
def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
|
||||
"""
|
||||
Collects current metrics from Prometheus and returns some of them
|
||||
in the form of the `named_metrics` list for `create_orca_header()`.
|
||||
|
||||
Parameters:
|
||||
- None
|
||||
|
||||
Returns:
|
||||
- list[tuple[str, float]]: List of tuples of metric names and their values.
|
||||
"""
|
||||
named_metrics: list[tuple[str, float]] = []
|
||||
# Map from prometheus metric names to ORCA named metrics.
|
||||
prometheus_to_orca_metrics = {
|
||||
"vllm:kv_cache_usage_perc": "kv_cache_usage_perc",
|
||||
"vllm:num_requests_waiting": "num_requests_waiting",
|
||||
}
|
||||
metrics = get_metrics_snapshot()
|
||||
for metric in metrics:
|
||||
orca_name = prometheus_to_orca_metrics.get(metric.name)
|
||||
# If this metric is mapped into ORCA, then add it to the report.
|
||||
# Note: Only Gauge metrics are currently supported.
|
||||
if orca_name is not None and isinstance(metric, Gauge):
|
||||
named_metrics.append((str(orca_name), float(metric.value)))
|
||||
return named_metrics
|
||||
|
||||
|
||||
def metrics_header(metrics_format: str) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
|
||||
Metrics are collected from Prometheus using `get_named_metrics_from_prometheus()`.
|
||||
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
"""
|
||||
if not metrics_format:
|
||||
return None
|
||||
# Get named metrics from prometheus.
|
||||
named_metrics = get_named_metrics_from_prometheus()
|
||||
return create_orca_header(metrics_format, named_metrics)
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RequestLogger:
|
||||
def __init__(self, *, max_log_len: int | None) -> None:
|
||||
self.max_log_len = max_log_len
|
||||
|
||||
if not logger.isEnabledFor(logging.INFO):
|
||||
logger.warning_once(
|
||||
"`--enable-log-requests` is set but "
|
||||
"the minimum log level is higher than INFO. "
|
||||
"No request information will be logged."
|
||||
)
|
||||
elif not logger.isEnabledFor(logging.DEBUG):
|
||||
logger.info_once(
|
||||
"`--enable-log-requests` is set but "
|
||||
"the minimum log level is higher than DEBUG. "
|
||||
"Only limited information will be logged to minimize overhead. "
|
||||
"To view more details, set `VLLM_LOGGING_LEVEL=DEBUG`."
|
||||
)
|
||||
|
||||
def log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: str | None,
|
||||
prompt_token_ids: list[int] | None,
|
||||
prompt_embeds: torch.Tensor | None,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
max_log_len = self.max_log_len
|
||||
if max_log_len is not None:
|
||||
if prompt is not None:
|
||||
prompt = prompt[:max_log_len]
|
||||
|
||||
if prompt_token_ids is not None:
|
||||
prompt_token_ids = prompt_token_ids[:max_log_len]
|
||||
|
||||
logger.debug(
|
||||
"Request %s details: prompt: %r, "
|
||||
"prompt_token_ids: %s, "
|
||||
"prompt_embeds shape: %s.",
|
||||
request_id,
|
||||
prompt,
|
||||
prompt_token_ids,
|
||||
prompt_embeds.shape if prompt_embeds is not None else None,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Received request %s: params: %s, lora_request: %s.",
|
||||
request_id,
|
||||
params,
|
||||
lora_request,
|
||||
)
|
||||
|
||||
def log_outputs(
|
||||
self,
|
||||
request_id: str,
|
||||
outputs: str,
|
||||
output_token_ids: Sequence[int] | None,
|
||||
finish_reason: str | None = None,
|
||||
is_streaming: bool = False,
|
||||
delta: bool = False,
|
||||
) -> None:
|
||||
max_log_len = self.max_log_len
|
||||
if max_log_len is not None:
|
||||
if outputs is not None:
|
||||
outputs = outputs[:max_log_len]
|
||||
|
||||
if output_token_ids is not None:
|
||||
# Convert to list and apply truncation
|
||||
output_token_ids = list(output_token_ids)[:max_log_len]
|
||||
|
||||
stream_info = ""
|
||||
if is_streaming:
|
||||
stream_info = " (streaming delta)" if delta else " (streaming complete)"
|
||||
|
||||
logger.info(
|
||||
"Generated response %s%s: output: %r, "
|
||||
"output_token_ids: %s, finish_reason: %s",
|
||||
request_id,
|
||||
stream_info,
|
||||
outputs,
|
||||
output_token_ids,
|
||||
finish_reason,
|
||||
)
|
||||
@@ -0,0 +1,479 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
|
||||
import pydantic
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
from starlette.datastructures import Headers, MutableHeaders
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import (
|
||||
create_error_response,
|
||||
sanitize_message,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
"""
|
||||
Pure ASGI middleware that authenticates each request by checking
|
||||
if the Authorization Bearer token exists and equals anyof "{api_key}".
|
||||
|
||||
Notes
|
||||
-----
|
||||
There are two cases in which authentication is skipped:
|
||||
1. The HTTP method is OPTIONS.
|
||||
2. The request path doesn't start with GUARDED_PREFIX (e.g. /health).
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
|
||||
self.app = app
|
||||
self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens]
|
||||
|
||||
def verify_token(self, headers: Headers) -> bool:
|
||||
authorization_header_value = headers.get("Authorization")
|
||||
if not authorization_header_value:
|
||||
return False
|
||||
|
||||
scheme, _, param = authorization_header_value.partition(" ")
|
||||
if scheme.lower() != "bearer":
|
||||
return False
|
||||
|
||||
param_hash = hashlib.sha256(param.encode("utf-8")).digest()
|
||||
|
||||
token_match = False
|
||||
for token_hash in self.api_tokens:
|
||||
token_match |= secrets.compare_digest(param_hash, token_hash)
|
||||
|
||||
return token_match
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if (
|
||||
scope["type"] not in ("http", "websocket")
|
||||
or scope.get("method") == "OPTIONS"
|
||||
):
|
||||
# scope["type"] can be "lifespan" or "startup" for example,
|
||||
# in which case we don't need to do anything
|
||||
return self.app(scope, receive, send)
|
||||
root_path = scope.get("root_path", "")
|
||||
url_path = scope["path"].removeprefix(root_path)
|
||||
headers = Headers(scope=scope)
|
||||
# Type narrow to satisfy mypy.
|
||||
if url_path.startswith(GUARDED_PREFIX) and not self.verify_token(headers):
|
||||
response = JSONResponse(content={"error": "Unauthorized"}, status_code=401)
|
||||
return response(scope, receive, send)
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
|
||||
class XRequestIdMiddleware:
|
||||
"""
|
||||
Middleware the set's the X-Request-Id header for each response
|
||||
to a random uuid4 (hex) value if the header isn't already
|
||||
present in the request, otherwise use the provided request id.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if scope["type"] not in ("http", "websocket"):
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
# Extract the request headers.
|
||||
request_headers = Headers(scope=scope)
|
||||
|
||||
async def send_with_request_id(message: Message) -> None:
|
||||
"""
|
||||
Custom send function to mutate the response headers
|
||||
and append X-Request-Id to it.
|
||||
"""
|
||||
if message["type"] == "http.response.start":
|
||||
response_headers = MutableHeaders(raw=message["headers"])
|
||||
request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex)
|
||||
response_headers.append("X-Request-Id", request_id)
|
||||
await send(message)
|
||||
|
||||
return self.app(scope, receive, send_with_request_id)
|
||||
|
||||
|
||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
if not log_config_file:
|
||||
return None
|
||||
try:
|
||||
with open(log_config_file) as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to load log config from file %s: error %s", log_config_file, e
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_uvicorn_log_config(args: Namespace) -> dict | None:
|
||||
"""
|
||||
Get the uvicorn log config based on the provided arguments.
|
||||
|
||||
Priority:
|
||||
1. If log_config_file is specified, use it
|
||||
2. If disable_access_log_for_endpoints is specified, create a config with
|
||||
the access log filter
|
||||
3. Otherwise, return None (use uvicorn defaults)
|
||||
"""
|
||||
# First, try to load from file if specified
|
||||
log_config = load_log_config(args.log_config_file)
|
||||
if log_config is not None:
|
||||
return log_config
|
||||
|
||||
# If endpoints to filter are specified, create a config with the filter
|
||||
if args.disable_access_log_for_endpoints:
|
||||
from vllm.logging_utils import create_uvicorn_log_config
|
||||
|
||||
# Parse comma-separated string into list
|
||||
excluded_paths = [
|
||||
p.strip()
|
||||
for p in args.disable_access_log_for_endpoints.split(",")
|
||||
if p.strip()
|
||||
]
|
||||
return create_uvicorn_log_config(
|
||||
excluded_paths=excluded_paths,
|
||||
log_level=args.uvicorn_log_level,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_content_from_chunk(chunk_data: dict) -> str:
|
||||
"""Extract content from a streaming response chunk."""
|
||||
try:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
|
||||
# Try using Completion types for type-safe parsing
|
||||
if chunk_data.get("object") == "chat.completion.chunk":
|
||||
chat_response = ChatCompletionStreamResponse.model_validate(chunk_data)
|
||||
if chat_response.choices and chat_response.choices[0].delta.content:
|
||||
return chat_response.choices[0].delta.content
|
||||
elif chunk_data.get("object") == "text_completion":
|
||||
completion_response = CompletionStreamResponse.model_validate(chunk_data)
|
||||
if completion_response.choices and completion_response.choices[0].text:
|
||||
return completion_response.choices[0].text
|
||||
except pydantic.ValidationError:
|
||||
# Fallback to manual parsing
|
||||
if "choices" in chunk_data and chunk_data["choices"]:
|
||||
choice = chunk_data["choices"][0]
|
||||
if "delta" in choice and choice["delta"].get("content"):
|
||||
return choice["delta"]["content"]
|
||||
elif choice.get("text"):
|
||||
return choice["text"]
|
||||
return ""
|
||||
|
||||
|
||||
class SSEDecoder:
|
||||
"""Robust Server-Sent Events decoder for streaming responses."""
|
||||
|
||||
def __init__(self):
|
||||
self.buffer = ""
|
||||
self.content_buffer = []
|
||||
|
||||
def decode_chunk(self, chunk: bytes) -> list[dict]:
|
||||
"""Decode a chunk of SSE data and return parsed events."""
|
||||
import json
|
||||
|
||||
try:
|
||||
chunk_str = chunk.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# Skip malformed chunks
|
||||
return []
|
||||
|
||||
self.buffer += chunk_str
|
||||
events = []
|
||||
|
||||
# Process complete lines
|
||||
while "\n" in self.buffer:
|
||||
line, self.buffer = self.buffer.split("\n", 1)
|
||||
line = line.rstrip("\r") # Handle CRLF
|
||||
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:].strip()
|
||||
if data_str == "[DONE]":
|
||||
events.append({"type": "done"})
|
||||
elif data_str:
|
||||
try:
|
||||
event_data = json.loads(data_str)
|
||||
events.append({"type": "data", "data": event_data})
|
||||
except json.JSONDecodeError:
|
||||
# Skip malformed JSON
|
||||
continue
|
||||
|
||||
return events
|
||||
|
||||
def extract_content(self, event_data: dict) -> str:
|
||||
"""Extract content from event data."""
|
||||
return _extract_content_from_chunk(event_data)
|
||||
|
||||
def add_content(self, content: str) -> None:
|
||||
"""Add content to the buffer."""
|
||||
if content:
|
||||
self.content_buffer.append(content)
|
||||
|
||||
def get_complete_content(self) -> str:
|
||||
"""Get the complete buffered content."""
|
||||
return "".join(self.content_buffer)
|
||||
|
||||
|
||||
def _log_streaming_response(response, response_body: list) -> None:
|
||||
"""Log streaming response with robust SSE parsing."""
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
|
||||
sse_decoder = SSEDecoder()
|
||||
chunk_count = 0
|
||||
|
||||
def buffered_iterator():
|
||||
nonlocal chunk_count
|
||||
|
||||
for chunk in response_body:
|
||||
chunk_count += 1
|
||||
yield chunk
|
||||
|
||||
# Parse SSE events from chunk
|
||||
events = sse_decoder.decode_chunk(chunk)
|
||||
|
||||
for event in events:
|
||||
if event["type"] == "data":
|
||||
content = sse_decoder.extract_content(event["data"])
|
||||
sse_decoder.add_content(content)
|
||||
elif event["type"] == "done":
|
||||
# Log complete content when done
|
||||
full_content = sse_decoder.get_complete_content()
|
||||
if full_content:
|
||||
# Truncate if too long
|
||||
if len(full_content) > 2048:
|
||||
full_content = full_content[:2048] + ""
|
||||
"...[truncated]"
|
||||
logger.info(
|
||||
"response_body={streaming_complete: content=%r, chunks=%d}",
|
||||
full_content,
|
||||
chunk_count,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"response_body={streaming_complete: no_content, chunks=%d}",
|
||||
chunk_count,
|
||||
)
|
||||
return
|
||||
|
||||
response.body_iterator = iterate_in_threadpool(buffered_iterator())
|
||||
logger.info("response_body={streaming_started: chunks=%d}", len(response_body))
|
||||
|
||||
|
||||
def _log_non_streaming_response(response_body: list) -> None:
|
||||
"""Log non-streaming response."""
|
||||
try:
|
||||
decoded_body = response_body[0].decode()
|
||||
logger.info("response_body={%s}", decoded_body)
|
||||
except UnicodeDecodeError:
|
||||
logger.info("response_body={<binary_data>}")
|
||||
|
||||
|
||||
async def log_response(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response_body = [section async for section in response.body_iterator]
|
||||
response.body_iterator = iterate_in_threadpool(iter(response_body))
|
||||
# Check if this is a streaming response by looking at content-type
|
||||
content_type = response.headers.get("content-type", "")
|
||||
is_streaming = content_type == "text/event-stream; charset=utf-8"
|
||||
|
||||
# Log response body based on type
|
||||
if not response_body:
|
||||
logger.info("response_body={<empty>}")
|
||||
elif is_streaming:
|
||||
_log_streaming_response(response, response_body)
|
||||
else:
|
||||
_log_non_streaming_response(response_body)
|
||||
return response
|
||||
|
||||
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
method. This error could be request specific (and therefore
|
||||
recoverable - e.g. if there is an error in input processing).
|
||||
|
||||
EngineDeadError is raised by the background output_handler
|
||||
method. This error is global and therefore not recoverable.
|
||||
|
||||
We register these @app.exception_handlers to return nice
|
||||
responses to the end user if they occur and shut down if needed.
|
||||
See https://fastapi.tiangolo.com/tutorial/handling-errors/
|
||||
for more details on how exception handlers work.
|
||||
|
||||
If an exception is encountered in a StreamingResponse
|
||||
generator, the exception is not raised, since we already sent
|
||||
a 200 status. Rather, we send an error message as the next chunk.
|
||||
Since the exception is not raised, this means that the server
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
terminate_if_errored(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.error(
|
||||
"Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def http_exception_handler(req: Request, exc: HTTPException):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"HTTPException caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(exc.detail),
|
||||
type=HTTPStatus(exc.status_code).phrase,
|
||||
code=exc.status_code,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=exc.status_code)
|
||||
|
||||
|
||||
async def validation_exception_handler(req: Request, exc: RequestValidationError):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"RequestValidationError caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
param = None
|
||||
errors = exc.errors()
|
||||
for error in errors:
|
||||
if "ctx" in error and "error" in error["ctx"]:
|
||||
ctx_error = error["ctx"]["error"]
|
||||
if isinstance(ctx_error, VLLMValidationError):
|
||||
param = ctx_error.parameter
|
||||
break
|
||||
|
||||
exc_str = str(exc)
|
||||
errors_str = str(errors)
|
||||
|
||||
if errors and errors_str and errors_str != exc_str:
|
||||
message = f"{exc_str} {errors_str}"
|
||||
else:
|
||||
message = exc_str
|
||||
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=HTTPStatus.BAD_REQUEST.phrase,
|
||||
code=HTTPStatus.BAD_REQUEST,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
_running_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
if app.state.log_stats:
|
||||
engine_client: EngineClient = app.state.engine_client
|
||||
|
||||
async def _force_log():
|
||||
while True:
|
||||
await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
|
||||
await engine_client.do_log_stats()
|
||||
|
||||
task = asyncio.create_task(_force_log())
|
||||
_running_tasks.add(task)
|
||||
task.add_done_callback(_running_tasks.remove)
|
||||
else:
|
||||
task = None
|
||||
|
||||
# Mark the startup heap as static so that it's ignored by GC.
|
||||
# Reduces pause times of oldest generation collections.
|
||||
freeze_gc_heap()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
finally:
|
||||
# Ensure app state including engine ref is gc'd
|
||||
del app.state
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from ssl import SSLContext
|
||||
|
||||
from watchfiles import Change, awatch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SSLCertRefresher:
|
||||
"""A class that monitors SSL certificate files and
|
||||
reloads them when they change.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ssl_context: SSLContext,
|
||||
key_path: str | None = None,
|
||||
cert_path: str | None = None,
|
||||
ca_path: str | None = None,
|
||||
) -> None:
|
||||
self.ssl = ssl_context
|
||||
self.key_path = key_path
|
||||
self.cert_path = cert_path
|
||||
self.ca_path = ca_path
|
||||
|
||||
# Setup certification chain watcher
|
||||
def update_ssl_cert_chain(change: Change, file_path: str) -> None:
|
||||
logger.info("Reloading SSL certificate chain")
|
||||
assert self.key_path and self.cert_path
|
||||
self.ssl.load_cert_chain(self.cert_path, self.key_path)
|
||||
|
||||
self.watch_ssl_cert_task = None
|
||||
if self.key_path and self.cert_path:
|
||||
self.watch_ssl_cert_task = asyncio.create_task(
|
||||
self._watch_files(
|
||||
[self.key_path, self.cert_path], update_ssl_cert_chain
|
||||
)
|
||||
)
|
||||
|
||||
# Setup CA files watcher
|
||||
def update_ssl_ca(change: Change, file_path: str) -> None:
|
||||
logger.info("Reloading SSL CA certificates")
|
||||
assert self.ca_path
|
||||
self.ssl.load_verify_locations(self.ca_path)
|
||||
|
||||
self.watch_ssl_ca_task = None
|
||||
if self.ca_path:
|
||||
self.watch_ssl_ca_task = asyncio.create_task(
|
||||
self._watch_files([self.ca_path], update_ssl_ca)
|
||||
)
|
||||
|
||||
async def _watch_files(self, paths, fun: Callable[[Change, str], None]) -> None:
|
||||
"""Watch multiple file paths asynchronously."""
|
||||
logger.info("SSLCertRefresher monitors files: %s", paths)
|
||||
async for changes in awatch(*paths):
|
||||
try:
|
||||
for change, file_path in changes:
|
||||
logger.info("File change detected: %s - %s", change.name, file_path)
|
||||
fun(change, file_path)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"SSLCertRefresher failed taking action on file change. Error: %s", e
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop watching files."""
|
||||
if self.watch_ssl_cert_task:
|
||||
self.watch_ssl_cert_task.cancel()
|
||||
self.watch_ssl_cert_task = None
|
||||
if self.watch_ssl_ca_task:
|
||||
self.watch_ssl_ca_task.cancel()
|
||||
self.watch_ssl_ca_task = None
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeVar
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
)
|
||||
|
||||
# Used internally
|
||||
_ChatCompletionResponseChoiceT = TypeVar(
|
||||
"_ChatCompletionResponseChoiceT",
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
)
|
||||
|
||||
|
||||
def maybe_filter_parallel_tool_calls(
|
||||
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
|
||||
) -> _ChatCompletionResponseChoiceT:
|
||||
"""Filter to first tool call only when parallel_tool_calls is False."""
|
||||
|
||||
if request.parallel_tool_calls:
|
||||
return choice
|
||||
|
||||
if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
|
||||
choice.message.tool_calls = choice.message.tool_calls[:1]
|
||||
elif (
|
||||
isinstance(choice, ChatCompletionResponseStreamChoice)
|
||||
and choice.delta.tool_calls
|
||||
):
|
||||
choice.delta.tool_calls = [
|
||||
tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
|
||||
]
|
||||
|
||||
return choice
|
||||
Reference in New Issue
Block a user