[Frontend] Consolidate online serving utils. (#44479)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -27,12 +27,22 @@ from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launcher import serve_http
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.server_utils import (
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
cli_env_setup,
|
||||
log_non_default_args,
|
||||
log_version_and_model,
|
||||
process_lora_modules,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
engine_error_handler,
|
||||
exception_handler,
|
||||
generation_error_handler,
|
||||
@@ -42,16 +52,6 @@ from vllm.entrypoints.openai.server_utils import (
|
||||
log_response,
|
||||
validation_exception_handler,
|
||||
)
|
||||
from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
|
||||
from vllm.entrypoints.utils import (
|
||||
cli_env_setup,
|
||||
log_non_default_args,
|
||||
log_version_and_model,
|
||||
process_lora_modules,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
@@ -187,7 +187,7 @@ def build_app(
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
from vllm.entrypoints.sagemaker.api_router import (
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
@@ -254,12 +254,12 @@ def build_app(
|
||||
|
||||
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
|
||||
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
|
||||
from vllm.entrypoints.openai.server_utils import AuthenticationMiddleware
|
||||
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
|
||||
|
||||
app.add_middleware(AuthenticationMiddleware, tokens=tokens)
|
||||
|
||||
if args.enable_request_id_headers:
|
||||
from vllm.entrypoints.openai.server_utils import XRequestIdMiddleware
|
||||
from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware
|
||||
|
||||
app.add_middleware(XRequestIdMiddleware)
|
||||
|
||||
|
||||
@@ -15,12 +15,12 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.orca_metrics import metrics_header
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.utils import (
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
@@ -21,7 +21,6 @@ from vllm.entrypoints.chat_utils import (
|
||||
get_tool_call_id_type,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
@@ -57,8 +56,11 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_streamable_parser_for_assistant,
|
||||
parse_chat_output,
|
||||
)
|
||||
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
)
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -20,11 +20,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
validate_chat_template,
|
||||
)
|
||||
from vllm.entrypoints.constants import (
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.entrypoints.serve.utils.constants import (
|
||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
@@ -13,12 +13,12 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.orca_metrics import metrics_header
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.utils import (
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.orca_metrics import metrics_header
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -13,7 +13,6 @@ import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionRequest,
|
||||
@@ -34,7 +33,8 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
clamp_prompt_logprobs,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -16,7 +16,6 @@ from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
@@ -39,12 +38,13 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
|
||||
from vllm.entrypoints.utils import create_error_response
|
||||
from vllm.inputs import EngineInput, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
@@ -153,7 +153,7 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
# Computed once at startup (cached by ``vllm_config`` identity) and
|
||||
# stamped on non-streaming responses. Streaming chunks deliberately
|
||||
# omit it to avoid per-chunk overhead.
|
||||
from vllm.entrypoints.openai.fingerprint import get_system_fingerprint
|
||||
from vllm.entrypoints.serve.utils.fingerprint import get_system_fingerprint
|
||||
|
||||
try:
|
||||
self.system_fingerprint: str | None = get_system_fingerprint(
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Build the ``system_fingerprint`` string returned by the OpenAI-compatible
|
||||
server.
|
||||
|
||||
Four modes, configured via ``--fingerprint-mode``:
|
||||
|
||||
* ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>`` — encodes
|
||||
server version, any non-trivial parallelism degree (tp/pp/dp/ep), and an
|
||||
8-char prefix of ``vllm_config.compute_hash()`` (covers model identity,
|
||||
quant config, speculative, attention backend, etc.).
|
||||
* ``hash``: ``vllm-<version>-<hash8>`` — parallelism stripped.
|
||||
* ``custom``: user-provided literal via ``--fingerprint-value``.
|
||||
* ``none``: the field is omitted (serialized as ``null``).
|
||||
|
||||
``get_system_fingerprint`` is only called at serving-class init (a handful
|
||||
of times per server); each subclass caches the returned string on
|
||||
``self.system_fingerprint``, so per-request cost is one attribute read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
FingerprintMode = Literal["full", "hash", "custom", "none"]
|
||||
|
||||
_DEFAULT_MODE: FingerprintMode = "full"
|
||||
_CUSTOM_VALUE: str | None = None
|
||||
|
||||
|
||||
def set_default_fingerprint_mode(
|
||||
mode: FingerprintMode,
|
||||
custom_value: str | None = None,
|
||||
) -> None:
|
||||
"""Configure the fingerprint mode for subsequent ``get_system_fingerprint``
|
||||
calls. Called once at server startup."""
|
||||
global _DEFAULT_MODE, _CUSTOM_VALUE
|
||||
_DEFAULT_MODE = mode
|
||||
_CUSTOM_VALUE = custom_value
|
||||
|
||||
|
||||
def get_system_fingerprint(vllm_config: Any) -> str | None:
|
||||
"""Return the fingerprint for ``vllm_config`` using the mode configured by
|
||||
``set_default_fingerprint_mode``."""
|
||||
return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
|
||||
|
||||
|
||||
def build_system_fingerprint(
|
||||
vllm_config: Any,
|
||||
mode: FingerprintMode = "full",
|
||||
custom_value: str | None = None,
|
||||
) -> str | None:
|
||||
if mode == "none":
|
||||
return None
|
||||
if mode == "custom":
|
||||
return custom_value
|
||||
|
||||
from vllm import __version__ as vllm_version
|
||||
|
||||
try:
|
||||
hash8 = vllm_config.compute_hash()[:8]
|
||||
except Exception:
|
||||
hash8 = "nohash"
|
||||
|
||||
if mode == "hash":
|
||||
return f"vllm-{vllm_version}-{hash8}"
|
||||
|
||||
# mode == "full"
|
||||
parts: list[str] = [f"vllm-{vllm_version}"]
|
||||
pc = getattr(vllm_config, "parallel_config", None)
|
||||
if pc is not None:
|
||||
tp = getattr(pc, "tensor_parallel_size", 1)
|
||||
if tp > 1:
|
||||
parts.append(f"tp{tp}")
|
||||
pp = getattr(pc, "pipeline_parallel_size", 1)
|
||||
if pp > 1:
|
||||
parts.append(f"pp{pp}")
|
||||
dp = getattr(pc, "data_parallel_size", 1)
|
||||
if dp > 1:
|
||||
parts.append(f"dp{dp}")
|
||||
if getattr(pc, "enable_expert_parallel", False):
|
||||
parts.append("ep")
|
||||
parts.append(hash8)
|
||||
return "-".join(parts)
|
||||
@@ -18,7 +18,7 @@ from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
)
|
||||
from vllm.entrypoints.utils import create_error_response
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.exceptions import LoRAAdapterNotFoundError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utility functions that create ORCA endpoint load report response headers.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.metrics.reader import Gauge, get_metrics_snapshot
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def create_orca_header(
|
||||
metrics_format: str, named_metrics: list[tuple[str, float]]
|
||||
) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format
|
||||
and adds custom metrics to named_metrics.
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
- named_metrics (List[Tuple[str, float]]): List of tuples with metric names
|
||||
and their corresponding double values.
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
"""
|
||||
|
||||
if metrics_format.lower() not in ["text", "json"]:
|
||||
logger.warning(
|
||||
"Warning: `%s` format is not supported in the ORCA response header",
|
||||
format,
|
||||
)
|
||||
return None
|
||||
|
||||
header = {}
|
||||
orca_report = {
|
||||
"named_metrics": {
|
||||
metric_name: value
|
||||
for metric_name, value in named_metrics
|
||||
if isinstance(metric_name, str) and isinstance(value, float)
|
||||
}
|
||||
}
|
||||
# output example:
|
||||
# endpoint-load-metrics: TEXT named_metrics.kv_cache_utilization=0.4
|
||||
if metrics_format.lower() == "text":
|
||||
native_http_header = ", ".join(
|
||||
[
|
||||
f"named_metrics.{metric_name}={value}"
|
||||
for metric_name, value in named_metrics
|
||||
if isinstance(metric_name, str) and isinstance(value, float)
|
||||
]
|
||||
)
|
||||
header["endpoint-load-metrics"] = f"TEXT {native_http_header}"
|
||||
|
||||
# output example:
|
||||
# endpoint-load-metrics: JSON “named_metrics”: {“custom-metric-util”: 0.4}
|
||||
elif metrics_format.lower() == "json":
|
||||
header["endpoint-load-metrics"] = f"JSON {json.dumps(orca_report)}"
|
||||
|
||||
logger.info("Created ORCA header %s", header)
|
||||
|
||||
return header
|
||||
|
||||
|
||||
def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
|
||||
"""
|
||||
Collects current metrics from Prometheus and returns some of them
|
||||
in the form of the `named_metrics` list for `create_orca_header()`.
|
||||
|
||||
Parameters:
|
||||
- None
|
||||
|
||||
Returns:
|
||||
- list[tuple[str, float]]: List of tuples of metric names and their values.
|
||||
"""
|
||||
named_metrics: list[tuple[str, float]] = []
|
||||
# Map from prometheus metric names to ORCA named metrics.
|
||||
prometheus_to_orca_metrics = {
|
||||
"vllm:kv_cache_usage_perc": "kv_cache_usage_perc",
|
||||
"vllm:num_requests_waiting": "num_requests_waiting",
|
||||
}
|
||||
metrics = get_metrics_snapshot()
|
||||
for metric in metrics:
|
||||
orca_name = prometheus_to_orca_metrics.get(metric.name)
|
||||
# If this metric is mapped into ORCA, then add it to the report.
|
||||
# Note: Only Gauge metrics are currently supported.
|
||||
if orca_name is not None and isinstance(metric, Gauge):
|
||||
named_metrics.append((str(orca_name), float(metric.value)))
|
||||
return named_metrics
|
||||
|
||||
|
||||
def metrics_header(metrics_format: str) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
|
||||
Metrics are collected from Prometheus using `get_named_metrics_from_prometheus()`.
|
||||
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
"""
|
||||
if not metrics_format:
|
||||
return None
|
||||
# Get named metrics from prometheus.
|
||||
named_metrics = get_named_metrics_from_prometheus()
|
||||
return create_orca_header(metrics_format, named_metrics)
|
||||
@@ -12,11 +12,11 @@ from openai.types.responses.response_output_message import ResponseOutputMessage
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.constants import MCP_PREFIX
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseInputOutputItem,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -15,9 +15,9 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
StreamingResponsesResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.utils import (
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm import envs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.constants import MCP_PREFIX
|
||||
from vllm.entrypoints.mcp.tool import Tool
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
@@ -40,6 +39,7 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
|
||||
from vllm.entrypoints.serve.utils.constants import MCP_PREFIX
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -32,7 +32,6 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
get_tool_call_id_type,
|
||||
)
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
@@ -93,7 +92,8 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.entrypoints.pooling.scoring.protocol import (
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
@@ -61,7 +62,6 @@ from vllm.entrypoints.speech_to_text.translation.protocol import (
|
||||
TranslationResponse,
|
||||
TranslationResponseVerbose,
|
||||
)
|
||||
from vllm.entrypoints.utils import create_error_response
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
|
||||
@@ -1,476 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
|
||||
import pydantic
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
from starlette.datastructures import Headers, MutableHeaders
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launcher import terminate_if_errored
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorInfo,
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.utils import create_error_response, sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
"""
|
||||
Pure ASGI middleware that authenticates each request by checking
|
||||
if the Authorization Bearer token exists and equals anyof "{api_key}".
|
||||
|
||||
Notes
|
||||
-----
|
||||
There are two cases in which authentication is skipped:
|
||||
1. The HTTP method is OPTIONS.
|
||||
2. The request path doesn't start with GUARDED_PREFIX (e.g. /health).
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
|
||||
self.app = app
|
||||
self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens]
|
||||
|
||||
def verify_token(self, headers: Headers) -> bool:
|
||||
authorization_header_value = headers.get("Authorization")
|
||||
if not authorization_header_value:
|
||||
return False
|
||||
|
||||
scheme, _, param = authorization_header_value.partition(" ")
|
||||
if scheme.lower() != "bearer":
|
||||
return False
|
||||
|
||||
param_hash = hashlib.sha256(param.encode("utf-8")).digest()
|
||||
|
||||
token_match = False
|
||||
for token_hash in self.api_tokens:
|
||||
token_match |= secrets.compare_digest(param_hash, token_hash)
|
||||
|
||||
return token_match
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if (
|
||||
scope["type"] not in ("http", "websocket")
|
||||
or scope.get("method") == "OPTIONS"
|
||||
):
|
||||
# scope["type"] can be "lifespan" or "startup" for example,
|
||||
# in which case we don't need to do anything
|
||||
return self.app(scope, receive, send)
|
||||
root_path = scope.get("root_path", "")
|
||||
url_path = scope["path"].removeprefix(root_path)
|
||||
headers = Headers(scope=scope)
|
||||
# Type narrow to satisfy mypy.
|
||||
if url_path.startswith(GUARDED_PREFIX) and not self.verify_token(headers):
|
||||
response = JSONResponse(content={"error": "Unauthorized"}, status_code=401)
|
||||
return response(scope, receive, send)
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
|
||||
class XRequestIdMiddleware:
|
||||
"""
|
||||
Middleware the set's the X-Request-Id header for each response
|
||||
to a random uuid4 (hex) value if the header isn't already
|
||||
present in the request, otherwise use the provided request id.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if scope["type"] not in ("http", "websocket"):
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
# Extract the request headers.
|
||||
request_headers = Headers(scope=scope)
|
||||
|
||||
async def send_with_request_id(message: Message) -> None:
|
||||
"""
|
||||
Custom send function to mutate the response headers
|
||||
and append X-Request-Id to it.
|
||||
"""
|
||||
if message["type"] == "http.response.start":
|
||||
response_headers = MutableHeaders(raw=message["headers"])
|
||||
request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex)
|
||||
response_headers.append("X-Request-Id", request_id)
|
||||
await send(message)
|
||||
|
||||
return self.app(scope, receive, send_with_request_id)
|
||||
|
||||
|
||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
if not log_config_file:
|
||||
return None
|
||||
try:
|
||||
with open(log_config_file) as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to load log config from file %s: error %s", log_config_file, e
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_uvicorn_log_config(args: Namespace) -> dict | None:
|
||||
"""
|
||||
Get the uvicorn log config based on the provided arguments.
|
||||
|
||||
Priority:
|
||||
1. If log_config_file is specified, use it
|
||||
2. If disable_access_log_for_endpoints is specified, create a config with
|
||||
the access log filter
|
||||
3. Otherwise, return None (use uvicorn defaults)
|
||||
"""
|
||||
# First, try to load from file if specified
|
||||
log_config = load_log_config(args.log_config_file)
|
||||
if log_config is not None:
|
||||
return log_config
|
||||
|
||||
# If endpoints to filter are specified, create a config with the filter
|
||||
if args.disable_access_log_for_endpoints:
|
||||
from vllm.logging_utils import create_uvicorn_log_config
|
||||
|
||||
# Parse comma-separated string into list
|
||||
excluded_paths = [
|
||||
p.strip()
|
||||
for p in args.disable_access_log_for_endpoints.split(",")
|
||||
if p.strip()
|
||||
]
|
||||
return create_uvicorn_log_config(
|
||||
excluded_paths=excluded_paths,
|
||||
log_level=args.uvicorn_log_level,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_content_from_chunk(chunk_data: dict) -> str:
|
||||
"""Extract content from a streaming response chunk."""
|
||||
try:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
|
||||
# Try using Completion types for type-safe parsing
|
||||
if chunk_data.get("object") == "chat.completion.chunk":
|
||||
chat_response = ChatCompletionStreamResponse.model_validate(chunk_data)
|
||||
if chat_response.choices and chat_response.choices[0].delta.content:
|
||||
return chat_response.choices[0].delta.content
|
||||
elif chunk_data.get("object") == "text_completion":
|
||||
completion_response = CompletionStreamResponse.model_validate(chunk_data)
|
||||
if completion_response.choices and completion_response.choices[0].text:
|
||||
return completion_response.choices[0].text
|
||||
except pydantic.ValidationError:
|
||||
# Fallback to manual parsing
|
||||
if "choices" in chunk_data and chunk_data["choices"]:
|
||||
choice = chunk_data["choices"][0]
|
||||
if "delta" in choice and choice["delta"].get("content"):
|
||||
return choice["delta"]["content"]
|
||||
elif choice.get("text"):
|
||||
return choice["text"]
|
||||
return ""
|
||||
|
||||
|
||||
class SSEDecoder:
|
||||
"""Robust Server-Sent Events decoder for streaming responses."""
|
||||
|
||||
def __init__(self):
|
||||
self.buffer = ""
|
||||
self.content_buffer = []
|
||||
|
||||
def decode_chunk(self, chunk: bytes) -> list[dict]:
|
||||
"""Decode a chunk of SSE data and return parsed events."""
|
||||
import json
|
||||
|
||||
try:
|
||||
chunk_str = chunk.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# Skip malformed chunks
|
||||
return []
|
||||
|
||||
self.buffer += chunk_str
|
||||
events = []
|
||||
|
||||
# Process complete lines
|
||||
while "\n" in self.buffer:
|
||||
line, self.buffer = self.buffer.split("\n", 1)
|
||||
line = line.rstrip("\r") # Handle CRLF
|
||||
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:].strip()
|
||||
if data_str == "[DONE]":
|
||||
events.append({"type": "done"})
|
||||
elif data_str:
|
||||
try:
|
||||
event_data = json.loads(data_str)
|
||||
events.append({"type": "data", "data": event_data})
|
||||
except json.JSONDecodeError:
|
||||
# Skip malformed JSON
|
||||
continue
|
||||
|
||||
return events
|
||||
|
||||
def extract_content(self, event_data: dict) -> str:
|
||||
"""Extract content from event data."""
|
||||
return _extract_content_from_chunk(event_data)
|
||||
|
||||
def add_content(self, content: str) -> None:
|
||||
"""Add content to the buffer."""
|
||||
if content:
|
||||
self.content_buffer.append(content)
|
||||
|
||||
def get_complete_content(self) -> str:
|
||||
"""Get the complete buffered content."""
|
||||
return "".join(self.content_buffer)
|
||||
|
||||
|
||||
def _log_streaming_response(response, response_body: list) -> None:
|
||||
"""Log streaming response with robust SSE parsing."""
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
|
||||
sse_decoder = SSEDecoder()
|
||||
chunk_count = 0
|
||||
|
||||
def buffered_iterator():
|
||||
nonlocal chunk_count
|
||||
|
||||
for chunk in response_body:
|
||||
chunk_count += 1
|
||||
yield chunk
|
||||
|
||||
# Parse SSE events from chunk
|
||||
events = sse_decoder.decode_chunk(chunk)
|
||||
|
||||
for event in events:
|
||||
if event["type"] == "data":
|
||||
content = sse_decoder.extract_content(event["data"])
|
||||
sse_decoder.add_content(content)
|
||||
elif event["type"] == "done":
|
||||
# Log complete content when done
|
||||
full_content = sse_decoder.get_complete_content()
|
||||
if full_content:
|
||||
# Truncate if too long
|
||||
if len(full_content) > 2048:
|
||||
full_content = full_content[:2048] + ""
|
||||
"...[truncated]"
|
||||
logger.info(
|
||||
"response_body={streaming_complete: content=%r, chunks=%d}",
|
||||
full_content,
|
||||
chunk_count,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"response_body={streaming_complete: no_content, chunks=%d}",
|
||||
chunk_count,
|
||||
)
|
||||
return
|
||||
|
||||
response.body_iterator = iterate_in_threadpool(buffered_iterator())
|
||||
logger.info("response_body={streaming_started: chunks=%d}", len(response_body))
|
||||
|
||||
|
||||
def _log_non_streaming_response(response_body: list) -> None:
|
||||
"""Log non-streaming response."""
|
||||
try:
|
||||
decoded_body = response_body[0].decode()
|
||||
logger.info("response_body={%s}", decoded_body)
|
||||
except UnicodeDecodeError:
|
||||
logger.info("response_body={<binary_data>}")
|
||||
|
||||
|
||||
async def log_response(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response_body = [section async for section in response.body_iterator]
|
||||
response.body_iterator = iterate_in_threadpool(iter(response_body))
|
||||
# Check if this is a streaming response by looking at content-type
|
||||
content_type = response.headers.get("content-type", "")
|
||||
is_streaming = content_type == "text/event-stream; charset=utf-8"
|
||||
|
||||
# Log response body based on type
|
||||
if not response_body:
|
||||
logger.info("response_body={<empty>}")
|
||||
elif is_streaming:
|
||||
_log_streaming_response(response, response_body)
|
||||
else:
|
||||
_log_non_streaming_response(response_body)
|
||||
return response
|
||||
|
||||
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
method. This error could be request specific (and therefore
|
||||
recoverable - e.g. if there is an error in input processing).
|
||||
|
||||
EngineDeadError is raised by the background output_handler
|
||||
method. This error is global and therefore not recoverable.
|
||||
|
||||
We register these @app.exception_handlers to return nice
|
||||
responses to the end user if they occur and shut down if needed.
|
||||
See https://fastapi.tiangolo.com/tutorial/handling-errors/
|
||||
for more details on how exception handlers work.
|
||||
|
||||
If an exception is encountered in a StreamingResponse
|
||||
generator, the exception is not raised, since we already sent
|
||||
a 200 status. Rather, we send an error message as the next chunk.
|
||||
Since the exception is not raised, this means that the server
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
terminate_if_errored(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.error(
|
||||
"Exception caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def http_exception_handler(req: Request, exc: HTTPException):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"HTTPException caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(exc.detail),
|
||||
type=HTTPStatus(exc.status_code).phrase,
|
||||
code=exc.status_code,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=exc.status_code)
|
||||
|
||||
|
||||
async def validation_exception_handler(req: Request, exc: RequestValidationError):
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"RequestValidationError caught. Request id: %s",
|
||||
req.state.request_metadata.request_id
|
||||
if hasattr(req.state, "request_metadata")
|
||||
else None,
|
||||
)
|
||||
|
||||
param = None
|
||||
errors = exc.errors()
|
||||
for error in errors:
|
||||
if "ctx" in error and "error" in error["ctx"]:
|
||||
ctx_error = error["ctx"]["error"]
|
||||
if isinstance(ctx_error, VLLMValidationError):
|
||||
param = ctx_error.parameter
|
||||
break
|
||||
|
||||
exc_str = str(exc)
|
||||
errors_str = str(errors)
|
||||
|
||||
if errors and errors_str and errors_str != exc_str:
|
||||
message = f"{exc_str} {errors_str}"
|
||||
else:
|
||||
message = exc_str
|
||||
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message=sanitize_message(message),
|
||||
type=HTTPStatus.BAD_REQUEST.phrase,
|
||||
code=HTTPStatus.BAD_REQUEST,
|
||||
param=param,
|
||||
)
|
||||
)
|
||||
return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
_running_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
if app.state.log_stats:
|
||||
engine_client: EngineClient = app.state.engine_client
|
||||
|
||||
async def _force_log():
|
||||
while True:
|
||||
await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
|
||||
await engine_client.do_log_stats()
|
||||
|
||||
task = asyncio.create_task(_force_log())
|
||||
_running_tasks.add(task)
|
||||
task.add_done_callback(_running_tasks.remove)
|
||||
else:
|
||||
task = None
|
||||
|
||||
# Mark the startup heap as static so that it's ignored by GC.
|
||||
# Reduces pause times of oldest generation collections.
|
||||
freeze_gc_heap()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
finally:
|
||||
# Ensure app state including engine ref is gc'd
|
||||
del app.state
|
||||
@@ -1,49 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
)
|
||||
|
||||
# Used internally
|
||||
_ChatCompletionResponseChoiceT = TypeVar(
|
||||
"_ChatCompletionResponseChoiceT",
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
)
|
||||
|
||||
|
||||
def maybe_filter_parallel_tool_calls(
|
||||
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
|
||||
) -> _ChatCompletionResponseChoiceT:
|
||||
"""Filter to first tool call only when parallel_tool_calls is False."""
|
||||
|
||||
if request.parallel_tool_calls:
|
||||
return choice
|
||||
|
||||
if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
|
||||
choice.message.tool_calls = choice.message.tool_calls[:1]
|
||||
elif (
|
||||
isinstance(choice, ChatCompletionResponseStreamChoice)
|
||||
and choice.delta.tool_calls
|
||||
):
|
||||
choice.delta.tool_calls = [
|
||||
tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
|
||||
]
|
||||
|
||||
return choice
|
||||
|
||||
|
||||
async def validate_json_request(raw_request: Request):
|
||||
content_type = raw_request.headers.get("content-type", "").lower()
|
||||
media_type = content_type.split(";", maxsplit=1)[0]
|
||||
if media_type != "application/json":
|
||||
raise RequestValidationError(
|
||||
errors=["Unsupported Media Type: Only 'application/json' is allowed"]
|
||||
)
|
||||
Reference in New Issue
Block a user