[openai api] log exception in exception handler (1/N) (#31164)

Signed-off-by: Andy Xie <andy.xning@gmail.com>
This commit is contained in:
Ning Xie
2026-03-05 16:00:12 +00:00
committed by GitHub
parent 612e7729c2
commit 176c799f4c
37 changed files with 908 additions and 1187 deletions
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock
import pytest import pytest
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
@@ -38,32 +37,6 @@ async def test_raise_if_error_raises_generation_error():
serving._raise_if_error(None, "test-request-id") # should not raise serving._raise_if_error(None, "test-request-id") # should not raise
@pytest.mark.asyncio
async def test_convert_generation_error_to_response():
"""test _convert_generation_error_to_response creates proper ErrorResponse"""
mock_engine = MagicMock()
mock_engine.model_config = MagicMock()
mock_engine.model_config.max_model_len = 100
mock_models = MagicMock()
serving = OpenAIServing(
engine_client=mock_engine,
models=mock_models,
request_logger=None,
)
# create a GenerationError
gen_error = GenerationError("Internal server error")
# convert to ErrorResponse
error_response = serving._convert_generation_error_to_response(gen_error)
assert isinstance(error_response, ErrorResponse)
assert error_response.error.type == "InternalServerError"
assert error_response.error.message == "Internal server error"
assert error_response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_convert_generation_error_to_streaming_response(): async def test_convert_generation_error_to_streaming_response():
"""test _convert_generation_error_to_streaming_response output""" """test _convert_generation_error_to_streaming_response output"""
@@ -13,7 +13,7 @@ from typing import Any
import pytest import pytest
import pytest_asyncio import pytest_asyncio
import requests import requests
from openai import BadRequestError, NotFoundError, OpenAI from openai import InternalServerError, NotFoundError, OpenAI
from openai_harmony import Message from openai_harmony import Message
from ....utils import RemoteOpenAIServer from ....utils import RemoteOpenAIServer
@@ -698,7 +698,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
async def test_function_calling_required(client: OpenAI, model_name: str): async def test_function_calling_required(client: OpenAI, model_name: str):
tools = [GET_WEATHER_SCHEMA] tools = [GET_WEATHER_SCHEMA]
with pytest.raises(BadRequestError): with pytest.raises(InternalServerError):
await client.responses.create( await client.responses.create(
model=model_name, model=model_name,
input="What's the weather like in Paris today?", input="What's the weather like in Paris today?",
+3 -8
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
@@ -11,7 +10,7 @@ import pytest
from vllm.config.multimodal import MultiModalConfig from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
@@ -145,12 +144,8 @@ async def test_chat_error_non_stream():
stream=False, stream=False,
) )
response = await serving_chat.create_chat_completion(request) with pytest.raises(GenerationError):
await serving_chat.create_chat_completion(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InternalServerError"
assert response.error.message == "Internal server error"
assert response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
@@ -11,7 +10,7 @@ import pytest
from vllm.config.multimodal import MultiModalConfig from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
@@ -131,12 +130,8 @@ async def test_completion_error_non_stream():
stream=False, stream=False,
) )
response = await serving_completion.create_completion(request) with pytest.raises(GenerationError):
await serving_completion.create_completion(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InternalServerError"
assert response.error.message == "Internal server error"
assert response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
@pytest.mark.asyncio @pytest.mark.asyncio
+28 -2
View File
@@ -1,12 +1,18 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json import json
from http import HTTPStatus
from typing import Final from typing import Final
import pytest import pytest
import schemathesis import schemathesis
from httpx import URL
from hypothesis import settings from hypothesis import settings
from schemathesis import GenerationConfig from schemathesis import GenerationConfig
from schemathesis.checks import not_a_server_error
from schemathesis.internal.checks import CheckContext
from schemathesis.models import Case
from schemathesis.transports.responses import GenericResponse
from ...utils import RemoteOpenAIServer from ...utils import RemoteOpenAIServer
@@ -127,10 +133,25 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
return strategy.filter(no_invalid_types) return strategy.filter(no_invalid_types)
def customized_not_a_server_error(
ctx: CheckContext, response: GenericResponse, case: Case
) -> bool | None:
try:
return not_a_server_error(ctx, response, case)
except Exception:
if (
URL(response.request.url).path
in ["/v1/chat/completions/render", "/v1/chat/completions"]
and response.status_code == HTTPStatus.NOT_IMPLEMENTED.value
):
return True
raise
@schema.parametrize() @schema.parametrize()
@schema.override(headers={"Content-Type": "application/json"}) @schema.override(headers={"Content-Type": "application/json"})
@settings(deadline=LONG_TIMEOUT_SECONDS * 1000, max_examples=50) @settings(deadline=LONG_TIMEOUT_SECONDS * 1000, max_examples=50)
def test_openapi_stateless(case: schemathesis.Case): def test_openapi_stateless(case: Case):
key = ( key = (
case.operation.method.upper(), case.operation.method.upper(),
case.operation.path, case.operation.path,
@@ -155,4 +176,9 @@ def test_openapi_stateless(case: schemathesis.Case):
}.get(key, DEFAULT_TIMEOUT_SECONDS) }.get(key, DEFAULT_TIMEOUT_SECONDS)
# No need to verify SSL certificate for localhost # No need to verify SSL certificate for localhost
case.call_and_validate(verify=False, timeout=timeout) case.call_and_validate(
verify=False,
timeout=timeout,
additional_checks=(customized_not_a_server_error,),
excluded_checks=(not_a_server_error,),
)
@@ -23,6 +23,7 @@ from vllm.entrypoints.openai.engine.protocol import (
) )
from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt from vllm.inputs import TokensPrompt
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer from vllm.renderers.hf import HfRenderer
@@ -818,9 +819,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
max_tokens=10, max_tokens=10,
) )
resp = await serving_chat.create_chat_completion(req) with pytest.raises(VLLMValidationError):
assert isinstance(resp, ErrorResponse) await serving_chat.create_chat_completion(req)
assert "context length is only" in resp.error.message
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -860,9 +860,8 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
max_tokens=1, max_tokens=1,
) )
resp = await serving_chat.create_chat_completion(req) with pytest.raises(VLLMValidationError):
assert isinstance(resp, ErrorResponse) await serving_chat.create_chat_completion(req)
assert "context length is only" in resp.error.message
@pytest.mark.asyncio @pytest.mark.asyncio
+3 -8
View File
@@ -17,9 +17,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse, ChatCompletionResponse,
) )
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.inputs import PromptType from vllm.inputs import PromptType
@@ -542,11 +539,9 @@ async def test_header_dp_rank_argument():
# Test 2: Out-of-range DP rank (1) # Test 2: Out-of-range DP rank (1)
mock_raw_request.headers = {"X-data-parallel-rank": "1"} mock_raw_request.headers = {"X-data-parallel-rank": "1"}
# should return ErrorResponse for out-of-range rank # should raise ValueError for out-of-range rank
response2 = await serving_chat.create_chat_completion(req, mock_raw_request) with pytest.raises(ValueError):
assert isinstance(response2, ErrorResponse), ( await serving_chat.create_chat_completion(req, mock_raw_request)
"Expected an ErrorResponse for out-of-range DP rank"
)
@pytest.mark.asyncio @pytest.mark.asyncio
+2 -41
View File
@@ -4,11 +4,10 @@
import asyncio import asyncio
import signal import signal
import socket import socket
from http import HTTPStatus
from typing import Any from typing import Any
import uvicorn import uvicorn
from fastapi import FastAPI, Request, Response from fastapi import FastAPI
from vllm import envs from vllm import envs
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
@@ -19,7 +18,6 @@ from vllm.entrypoints.constants import (
from vllm.entrypoints.ssl import SSLCertRefresher from vllm.entrypoints.ssl import SSLCertRefresher
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.network_utils import find_process_using_port from vllm.utils.network_utils import find_process_using_port
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -75,7 +73,7 @@ async def serve_http(
config.h11_max_header_count = h11_max_header_count config.h11_max_header_count = h11_max_header_count
config.load() config.load()
server = uvicorn.Server(config) server = uvicorn.Server(config)
_add_shutdown_handlers(app, server) app.state.server = server
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
@@ -148,40 +146,3 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
engine_errored = engine.errored and not engine.is_running engine_errored = engine.errored and not engine.is_running
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored: if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
server.should_exit = True server.should_exit = True
def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
"""
VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
method. This error could be request specific (and therefore
recoverable - e.g. if there is an error in input processing).
EngineDeadError is raised by the background output_handler
method. This error is global and therefore not recoverable.
We register these @app.exception_handlers to return nice
responses to the end user if they occur and shut down if needed.
See https://fastapi.tiangolo.com/tutorial/handling-errors/
for more details on how exception handlers work.
If an exception is encountered in a StreamingResponse
generator, the exception is not raised, since we already sent
a 200 status. Rather, we send an error message as the next chunk.
Since the exception is not raised, this means that the server
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
@app.exception_handler(RuntimeError)
@app.exception_handler(EngineDeadError)
@app.exception_handler(EngineGenerateError)
async def runtime_exception_handler(request: Request, __):
terminate_if_errored(
server=server,
engine=request.app.state.engine_client,
)
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+6 -1
View File
@@ -31,6 +31,8 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.server_utils import ( from vllm.entrypoints.openai.server_utils import (
engine_error_handler,
exception_handler,
get_uvicorn_log_config, get_uvicorn_log_config,
http_exception_handler, http_exception_handler,
lifespan, lifespan,
@@ -57,6 +59,7 @@ from vllm.usage.usage_lib import UsageContext
from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import is_valid_ipv6_address from vllm.utils.network_utils import is_valid_ipv6_address
from vllm.utils.system_utils import decorate_logs, set_ulimit from vllm.utils.system_utils import decorate_logs, set_ulimit
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm.version import __version__ as VLLM_VERSION from vllm.version import __version__ as VLLM_VERSION
prometheus_multiproc_dir: tempfile.TemporaryDirectory prometheus_multiproc_dir: tempfile.TemporaryDirectory
@@ -250,6 +253,9 @@ def build_app(
app.exception_handler(HTTPException)(http_exception_handler) app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(RequestValidationError)(validation_exception_handler) app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(EngineGenerateError)(engine_error_handler)
app.exception_handler(EngineDeadError)(engine_error_handler)
app.exception_handler(Exception)(exception_handler)
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY # Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]: if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
@@ -355,7 +361,6 @@ async def init_app_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
log_error_stack=args.log_error_stack,
) )
if any(task in supported_tasks for task in ("generate", "render")): if any(task in supported_tasks for task in ("generate", "render")):
@@ -39,6 +39,7 @@ def chat(request: Request) -> OpenAIServingChat | None:
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
}, },
) )
@with_cancellation @with_cancellation
@@ -54,10 +55,7 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
message="The model does not support Chat Completions API" message="The model does not support Chat Completions API"
) )
try: generator = await handler.create_chat_completion(request, raw_request)
generator = await handler.create_chat_completion(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -81,6 +79,7 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
}, },
) )
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request): async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
@@ -93,10 +92,7 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
message="The model does not support Chat Completions API" message="The model does not support Chat Completions API"
) )
try: result = await handler.render_chat_request(request)
result = await handler.render_chat_request(request)
except Exception as e:
result = handler.create_error_response(e)
if isinstance(result, ErrorResponse): if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+145 -175
View File
@@ -8,7 +8,6 @@ from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence from collections.abc import Sequence as GenericSequence
from typing import Any, Final from typing import Any, Final
import jinja2
import partial_json_parser import partial_json_parser
import regex as re import regex as re
from fastapi import Request from fastapi import Request
@@ -105,7 +104,6 @@ class OpenAIServingChat(OpenAIServing):
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
enable_log_outputs: bool = False, enable_log_outputs: bool = False,
enable_log_deltas: bool = True, enable_log_deltas: bool = True,
log_error_stack: bool = False,
default_chat_template_kwargs: dict[str, Any] | None = None, default_chat_template_kwargs: dict[str, Any] | None = None,
) -> None: ) -> None:
super().__init__( super().__init__(
@@ -113,7 +111,6 @@ class OpenAIServingChat(OpenAIServing):
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
log_error_stack=log_error_stack,
) )
self.response_role = response_role self.response_role = response_role
@@ -235,81 +232,76 @@ class OpenAIServingChat(OpenAIServing):
if self.engine_client.errored: if self.engine_client.errored:
raise self.engine_client.dead_error raise self.engine_client.dead_error
try: tokenizer = self.renderer.tokenizer
tokenizer = self.renderer.tokenizer
tool_parser = self.tool_parser tool_parser = self.tool_parser
if is_mistral_tokenizer(tokenizer): if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially # because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request # re-serialize the tool_calls field of the request
# for more info: see comment in `maybe_serialize_tool_calls` # for more info: see comment in `maybe_serialize_tool_calls`
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type] _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
_mt.validate_request_params(request) _mt.validate_request_params(request)
# Check if tool parsing is unavailable (common condition) # Check if tool parsing is unavailable (common condition)
tool_parsing_unavailable = ( tool_parsing_unavailable = (
tool_parser is None tool_parser is None
and not is_mistral_tokenizer(tokenizer) and not is_mistral_tokenizer(tokenizer)
and not self.use_harmony and not self.use_harmony
)
# Validate tool_choice when tool parsing is required but unavailable
if tool_parsing_unavailable and request.tool_choice not in (
None,
"none",
):
if request.tool_choice == "auto" and not self.enable_auto_tools:
# for hf tokenizers, "auto" tools requires
# --enable-auto-tool-choice and --tool-call-parser
return self.create_error_response(
'"auto" tool choice requires '
"--enable-auto-tool-choice and --tool-call-parser to be set"
)
elif request.tool_choice != "auto":
# "required" or named tool requires tool parser
return self.create_error_response(
f'tool_choice="{request.tool_choice}" requires '
"--tool-call-parser to be set"
)
if request.tools is None or (
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
):
tool_dicts = None
else:
tool_dicts = [tool.model_dump() for tool in request.tools]
if not self.use_harmony:
# Common case.
error_check_ret = self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
) )
if error_check_ret is not None:
return error_check_ret
# Validate tool_choice when tool parsing is required but unavailable conversation, engine_prompts = await self._preprocess_chat(
if tool_parsing_unavailable and request.tool_choice not in ( request,
None, request.messages,
"none", default_template=self.chat_template,
): default_template_content_format=self.chat_template_content_format,
if request.tool_choice == "auto" and not self.enable_auto_tools: default_template_kwargs=self.default_chat_template_kwargs,
# for hf tokenizers, "auto" tools requires tool_dicts=tool_dicts,
# --enable-auto-tool-choice and --tool-call-parser tool_parser=tool_parser,
return self.create_error_response( )
'"auto" tool choice requires ' else:
"--enable-auto-tool-choice and --tool-call-parser to be set" # For GPT-OSS.
) should_include_tools = tool_dicts is not None
elif request.tool_choice != "auto": conversation, engine_prompts = self._make_request_with_harmony(
# "required" or named tool requires tool parser request, should_include_tools
return self.create_error_response( )
f'tool_choice="{request.tool_choice}" requires '
"--tool-call-parser to be set"
)
if request.tools is None or (
request.tool_choice == "none"
and self.exclude_tools_when_tool_choice_none
):
tool_dicts = None
else:
tool_dicts = [tool.model_dump() for tool in request.tools]
if not self.use_harmony:
# Common case.
error_check_ret = self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
conversation, engine_prompts = await self._preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=self.default_chat_template_kwargs,
tool_dicts=tool_dicts,
tool_parser=tool_parser,
)
else:
# For GPT-OSS.
should_include_tools = tool_dicts is not None
conversation, engine_prompts = self._make_request_with_harmony(
request, should_include_tools
)
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(e)
return conversation, engine_prompts return conversation, engine_prompts
@@ -329,20 +321,16 @@ class OpenAIServingChat(OpenAIServing):
tokenizer = self.renderer.tokenizer tokenizer = self.renderer.tokenizer
assert tokenizer is not None assert tokenizer is not None
reasoning_parser: ReasoningParser | None = None reasoning_parser: ReasoningParser | None = None
try: if self.reasoning_parser_cls:
if self.reasoning_parser_cls: # Pass the same chat template kwargs as used in tokenization
# Pass the same chat template kwargs as used in tokenization chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
chat_template_kwargs = self._prepare_extra_chat_template_kwargs( request.chat_template_kwargs,
request.chat_template_kwargs, self.default_chat_template_kwargs,
self.default_chat_template_kwargs, )
) reasoning_parser = self.reasoning_parser_cls(
reasoning_parser = self.reasoning_parser_cls( tokenizer,
tokenizer, chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg] )
)
except RuntimeError as e:
logger.exception("Error in reasoning parser creation.")
return self.create_error_response(str(e))
result = await self.render_chat_request(request) result = await self.render_chat_request(request)
if isinstance(result, ErrorResponse): if isinstance(result, ErrorResponse):
return result return result
@@ -357,15 +345,9 @@ class OpenAIServingChat(OpenAIServing):
if raw_request: if raw_request:
raw_request.state.request_metadata = request_metadata raw_request.state.request_metadata = request_metadata
try: lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
lora_request = self._maybe_get_adapters(
request, supports_default_mm_loras=True
)
model_name = self.models.model_name(lora_request) model_name = self.models.model_name(lora_request)
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
# Extract data_parallel_rank from header (router can inject it) # Extract data_parallel_rank from header (router can inject it)
data_parallel_rank = self._get_data_parallel_rank(raw_request) data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -373,81 +355,76 @@ class OpenAIServingChat(OpenAIServing):
# Schedule the request and get the result generator. # Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = [] generators: list[AsyncGenerator[RequestOutput, None]] = []
try: for i, engine_prompt in enumerate(engine_prompts):
for i, engine_prompt in enumerate(engine_prompts): prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids
prompt_token_ids = self._extract_prompt_components(
engine_prompt
).token_ids
# If we are creating sub requests for multiple prompts, ensure that they # If we are creating sub requests for multiple prompts, ensure that they
# have unique request ids. # have unique request ids.
sub_request_id = ( sub_request_id = (
request_id if len(engine_prompts) == 1 else f"{request_id}_{i}" request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
)
max_tokens = get_max_tokens(
max_model_len,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
self._extract_prompt_len(engine_prompt),
self.default_sampling_params,
self.override_max_tokens,
)
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
) )
else:
max_tokens = get_max_tokens( sampling_params = request.to_sampling_params(
max_model_len, max_tokens,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
self._extract_prompt_len(engine_prompt),
self.default_sampling_params, self.default_sampling_params,
self.override_max_tokens,
) )
sampling_params: SamplingParams | BeamSearchParams self._log_inputs(
if request.use_beam_search: sub_request_id,
sampling_params = request.to_beam_search_params( engine_prompt,
max_tokens, self.default_sampling_params params=sampling_params,
) lora_request=lora_request,
else: )
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params,
)
self._log_inputs( trace_headers = (
sub_request_id, None
engine_prompt, if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
request_id=sub_request_id,
params=sampling_params, params=sampling_params,
lora_request=lora_request, lora_request=lora_request,
trace_headers=trace_headers,
)
else:
reasoning_ended = (
reasoning_parser.is_reasoning_end(prompt_token_ids or [])
if reasoning_parser
else None
) )
trace_headers = ( generator = self.engine_client.generate(
None engine_prompt,
if raw_request is None sampling_params,
else await self._get_trace_headers(raw_request.headers) sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
reasoning_ended=reasoning_ended,
) )
if isinstance(sampling_params, BeamSearchParams): generators.append(generator)
generator = self.beam_search(
prompt=engine_prompt,
request_id=sub_request_id,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
reasoning_ended = (
reasoning_parser.is_reasoning_end(prompt_token_ids or [])
if reasoning_parser
else None
)
generator = self.engine_client.generate(
engine_prompt,
sampling_params,
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
reasoning_ended=reasoning_ended,
)
generators.append(generator)
except ValueError as e:
return self.create_error_response(e)
assert len(generators) == 1 assert len(generators) == 1
(result_generator,) = generators (result_generator,) = generators
@@ -464,21 +441,16 @@ class OpenAIServingChat(OpenAIServing):
reasoning_parser, reasoning_parser,
) )
try: return await self.chat_completion_full_generator(
return await self.chat_completion_full_generator( request,
request, result_generator,
result_generator, request_id,
request_id, model_name,
model_name, conversation,
conversation, tokenizer,
tokenizer, request_metadata,
request_metadata, reasoning_parser,
reasoning_parser, )
)
except GenerationError as e:
return self._convert_generation_error_to_response(e)
except ValueError as e:
return self.create_error_response(e)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str: def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
if request.add_generation_prompt: if request.add_generation_prompt:
@@ -1414,8 +1386,6 @@ class OpenAIServingChat(OpenAIServing):
final_res = res final_res = res
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
assert final_res is not None assert final_res is not None
@@ -54,10 +54,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
message="The model does not support Completions API" message="The model does not support Completions API"
) )
try: generator = await handler.create_completion(request, raw_request)
generator = await handler.create_completion(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -91,10 +88,7 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
message="The model does not support Completions API" message="The model does not support Completions API"
) )
try: result = await handler.render_completion_request(request)
result = await handler.render_completion_request(request)
except Exception as e:
result = handler.create_error_response(e)
if isinstance(result, ErrorResponse): if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+52 -70
View File
@@ -7,7 +7,6 @@ from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence from collections.abc import Sequence as GenericSequence
from typing import cast from typing import cast
import jinja2
from fastapi import Request from fastapi import Request
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
@@ -56,14 +55,12 @@ class OpenAIServingCompletion(OpenAIServing):
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False, enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
log_error_stack: bool = False,
): ):
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
log_error_stack=log_error_stack,
) )
self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -110,15 +107,11 @@ class OpenAIServingCompletion(OpenAIServing):
"prompt_logprobs is not compatible with prompt embeds." "prompt_logprobs is not compatible with prompt embeds."
) )
try: engine_prompts = await self._preprocess_completion(
engine_prompts = await self._preprocess_completion( request,
request, prompt_input=request.prompt,
prompt_input=request.prompt, prompt_embeds=request.prompt_embeds,
prompt_embeds=request.prompt_embeds, )
)
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(e)
return engine_prompts return engine_prompts
@@ -149,11 +142,7 @@ class OpenAIServingCompletion(OpenAIServing):
if raw_request: if raw_request:
raw_request.state.request_metadata = request_metadata raw_request.state.request_metadata = request_metadata
try: lora_request = self._maybe_get_adapters(request)
lora_request = self._maybe_get_adapters(request)
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
# Extract data_parallel_rank from header (router can inject it) # Extract data_parallel_rank from header (router can inject it)
data_parallel_rank = self._get_data_parallel_rank(raw_request) data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -161,64 +150,61 @@ class OpenAIServingCompletion(OpenAIServing):
# Schedule the request and get the result generator. # Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len max_model_len = self.model_config.max_model_len
generators: list[AsyncGenerator[RequestOutput, None]] = [] generators: list[AsyncGenerator[RequestOutput, None]] = []
try: for i, engine_prompt in enumerate(engine_prompts):
for i, engine_prompt in enumerate(engine_prompts): max_tokens = get_max_tokens(
max_tokens = get_max_tokens( max_model_len,
max_model_len, request.max_tokens,
request.max_tokens, self._extract_prompt_len(engine_prompt),
self._extract_prompt_len(engine_prompt), self.default_sampling_params,
self.override_max_tokens,
)
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
)
else:
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params, self.default_sampling_params,
self.override_max_tokens,
) )
sampling_params: SamplingParams | BeamSearchParams request_id_item = f"{request_id}-{i}"
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
)
else:
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params,
)
request_id_item = f"{request_id}-{i}" self._log_inputs(
request_id_item,
engine_prompt,
params=sampling_params,
lora_request=lora_request,
)
self._log_inputs( trace_headers = (
request_id_item, None
engine_prompt, if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
request_id=request_id,
params=sampling_params, params=sampling_params,
lora_request=lora_request, lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_prompt,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
) )
trace_headers = ( generators.append(generator)
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
request_id=request_id,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_prompt,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
)
generators.append(generator)
except ValueError as e:
return self.create_error_response(e)
result_generator = merge_async_iterators(*generators) result_generator = merge_async_iterators(*generators)
@@ -273,10 +259,6 @@ class OpenAIServingCompletion(OpenAIServing):
) )
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except GenerationError as e:
return self._convert_generation_error_to_response(e)
except ValueError as e:
return self.create_error_response(e)
# When user requests streaming but we don't stream, we still need to # When user requests streaming but we don't stream, we still need to
# return a streaming response with a single event. # return a streaming response with a single event.
@@ -4,6 +4,7 @@
# Adapted from # Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
import time import time
from http import HTTPStatus
from typing import Any, ClassVar, Literal, TypeAlias from typing import Any, ClassVar, Literal, TypeAlias
import regex as re import regex as re
@@ -262,6 +263,14 @@ class DeltaMessage(OpenAIBaseModel):
tool_calls: list[DeltaToolCall] = Field(default_factory=list) tool_calls: list[DeltaToolCall] = Field(default_factory=list)
class GenerationError(Exception):
"""raised when finish_reason indicates internal server error (500)"""
def __init__(self, message: str = "Internal server error"):
super().__init__(message)
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
####### Tokens IN <> Tokens OUT ####### ####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel): class GenerateRequest(BaseModel):
request_id: str = Field( request_id: str = Field(
+51 -129
View File
@@ -2,9 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio import asyncio
import json import json
import sys
import time import time
import traceback
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
from dataclasses import dataclass, field from dataclasses import dataclass, field
from http import HTTPStatus from http import HTTPStatus
@@ -38,10 +36,10 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse, CompletionResponse,
) )
from vllm.entrypoints.openai.engine.protocol import ( from vllm.entrypoints.openai.engine.protocol import (
ErrorInfo,
ErrorResponse, ErrorResponse,
FunctionCall, FunctionCall,
FunctionDefinition, FunctionDefinition,
GenerationError,
) )
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.responses.context import ( from vllm.entrypoints.openai.responses.context import (
@@ -89,7 +87,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeCompletionRequest, TokenizeCompletionRequest,
TokenizeResponse, TokenizeResponse,
) )
from vllm.entrypoints.utils import get_max_tokens, sanitize_message from vllm.entrypoints.utils import create_error_response, get_max_tokens
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.inputs.data import ( from vllm.inputs.data import (
ProcessorInputs, ProcessorInputs,
@@ -125,15 +123,6 @@ from vllm.utils.async_utils import (
) )
from vllm.utils.mistral import is_mistral_tokenizer from vllm.utils.mistral import is_mistral_tokenizer
class GenerationError(Exception):
"""raised when finish_reason indicates internal server error (500)"""
def __init__(self, message: str = "Internal server error"):
super().__init__(message)
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -225,7 +214,6 @@ class OpenAIServing:
*, *,
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
): ):
super().__init__() super().__init__()
@@ -236,8 +224,6 @@ class OpenAIServing:
self.request_logger = request_logger self.request_logger = request_logger
self.return_tokens_as_token_ids = return_tokens_as_token_ids self.return_tokens_as_token_ids = return_tokens_as_token_ids
self.log_error_stack = log_error_stack
self.model_config = engine_client.model_config self.model_config = engine_client.model_config
self.renderer = engine_client.renderer self.renderer = engine_client.renderer
self.io_processor = engine_client.io_processor self.io_processor = engine_client.io_processor
@@ -526,133 +512,79 @@ class OpenAIServing:
"""Schedule the request and get the result generator.""" """Schedule the request and get the result generator."""
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
try: trace_headers = (
trace_headers = ( None
None if ctx.raw_request is None
if ctx.raw_request is None else await self._get_trace_headers(ctx.raw_request.headers)
else await self._get_trace_headers(ctx.raw_request.headers) )
pooling_params = self._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
if ctx.engine_prompts is None:
return self.create_error_response("Engine prompts not available")
for i, engine_prompt in enumerate(ctx.engine_prompts):
request_id_item = f"{ctx.request_id}-{i}"
self._log_inputs(
request_id_item,
engine_prompt,
params=pooling_params,
lora_request=ctx.lora_request,
) )
pooling_params = self._create_pooling_params(ctx) generator = self.engine_client.encode(
if isinstance(pooling_params, ErrorResponse): engine_prompt,
return pooling_params pooling_params,
request_id_item,
lora_request=ctx.lora_request,
trace_headers=trace_headers,
priority=getattr(ctx.request, "priority", 0),
)
if ctx.engine_prompts is None: generators.append(generator)
return self.create_error_response("Engine prompts not available")
for i, engine_prompt in enumerate(ctx.engine_prompts): ctx.result_generator = merge_async_iterators(*generators)
request_id_item = f"{ctx.request_id}-{i}"
self._log_inputs( return None
request_id_item,
engine_prompt,
params=pooling_params,
lora_request=ctx.lora_request,
)
generator = self.engine_client.encode(
engine_prompt,
pooling_params,
request_id_item,
lora_request=ctx.lora_request,
trace_headers=trace_headers,
priority=getattr(ctx.request, "priority", 0),
)
generators.append(generator)
ctx.result_generator = merge_async_iterators(*generators)
return None
except Exception as e:
return self.create_error_response(e)
async def _collect_batch( async def _collect_batch(
self, self,
ctx: ServeContext, ctx: ServeContext,
) -> ErrorResponse | None: ) -> ErrorResponse | None:
"""Collect batch results from the result generator.""" """Collect batch results from the result generator."""
try: if ctx.engine_prompts is None:
if ctx.engine_prompts is None: return self.create_error_response("Engine prompts not available")
return self.create_error_response("Engine prompts not available")
num_prompts = len(ctx.engine_prompts) num_prompts = len(ctx.engine_prompts)
final_res_batch: list[PoolingRequestOutput | None] final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_prompts final_res_batch = [None] * num_prompts
if ctx.result_generator is None: if ctx.result_generator is None:
return self.create_error_response("Result generator not available") return self.create_error_response("Result generator not available")
async for i, res in ctx.result_generator: async for i, res in ctx.result_generator:
final_res_batch[i] = res final_res_batch[i] = res
if None in final_res_batch: if None in final_res_batch:
return self.create_error_response( return self.create_error_response(
"Failed to generate results for all prompts" "Failed to generate results for all prompts"
) )
ctx.final_res_batch = [res for res in final_res_batch if res is not None] ctx.final_res_batch = [res for res in final_res_batch if res is not None]
return None return None
except Exception as e:
return self.create_error_response(e)
@staticmethod
def create_error_response( def create_error_response(
self,
message: str | Exception, message: str | Exception,
err_type: str = "BadRequestError", err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
param: str | None = None, param: str | None = None,
) -> ErrorResponse: ) -> ErrorResponse:
exc: Exception | None = None return create_error_response(message, err_type, status_code, param)
if isinstance(message, Exception):
exc = message
from vllm.exceptions import VLLMValidationError
if isinstance(exc, VLLMValidationError):
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = exc.parameter
elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
# Common validation errors from user input
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = None
elif isinstance(exc, NotImplementedError):
err_type = "NotImplementedError"
status_code = HTTPStatus.NOT_IMPLEMENTED
param = None
elif exc.__class__.__name__ == "TemplateError":
# jinja2.TemplateError (avoid importing jinja2)
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = None
else:
err_type = "InternalServerError"
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
param = None
message = str(exc)
if self.log_error_stack:
exc_type, _, _ = sys.exc_info()
if exc_type is not None:
traceback.print_exc()
else:
traceback.print_stack()
return ErrorResponse(
error=ErrorInfo(
message=sanitize_message(message),
type=err_type,
code=status_code.value,
param=param,
)
)
def create_streaming_error_response( def create_streaming_error_response(
self, self,
@@ -680,16 +612,6 @@ class OpenAIServing:
) )
raise GenerationError("Internal server error") raise GenerationError("Internal server error")
def _convert_generation_error_to_response(
self, e: GenerationError
) -> ErrorResponse:
"""Convert GenerationError to ErrorResponse."""
return self.create_error_response(
str(e),
err_type="InternalServerError",
status_code=e.status_code,
)
def _convert_generation_error_to_streaming_response( def _convert_generation_error_to_streaming_response(
self, e: GenerationError self, e: GenerationError
) -> str: ) -> str:
@@ -87,7 +87,6 @@ async def init_generate_state(
enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs, enable_log_outputs=args.enable_log_outputs,
log_error_stack=args.log_error_stack,
) )
if "generate" in supported_tasks if "generate" in supported_tasks
else None else None
@@ -111,7 +110,6 @@ async def init_generate_state(
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs, enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas, enable_log_deltas=args.enable_log_deltas,
log_error_stack=args.log_error_stack,
) )
if any(task in supported_tasks for task in ("generate", "render")) if any(task in supported_tasks for task in ("generate", "render"))
else None else None
@@ -127,7 +125,6 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids, return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
log_error_stack=args.log_error_stack,
) )
if any(task in supported_tasks for task in ("generate", "render")) if any(task in supported_tasks for task in ("generate", "render"))
else None else None
@@ -156,7 +153,6 @@ async def init_generate_state(
state.openai_serving_models, state.openai_serving_models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids, return_tokens_as_token_ids=args.return_tokens_as_token_ids,
log_error_stack=args.log_error_stack,
enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs, enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only, force_no_detokenize=args.tokens_only,
@@ -68,7 +68,6 @@ def init_realtime_state(
engine_client, engine_client,
state.openai_serving_models, state.openai_serving_models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=args.log_error_stack,
) )
if "realtime" in supported_tasks if "realtime" in supported_tasks
else None else None
@@ -33,13 +33,11 @@ class OpenAIServingRealtime(OpenAIServing):
models: OpenAIServingModels, models: OpenAIServingModels,
*, *,
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
log_error_stack: bool = False,
): ):
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=log_error_stack,
) )
self.task_type: Literal["realtime"] = "realtime" self.task_type: Literal["realtime"] = "realtime"
@@ -63,10 +63,8 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
return base_server.create_error_response( return base_server.create_error_response(
message="The model does not support Responses API" message="The model does not support Responses API"
) )
try:
generator = await handler.create_responses(request, raw_request) generator = await handler.create_responses(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -95,14 +93,11 @@ async def retrieve_responses(
message="The model does not support Responses API" message="The model does not support Responses API"
) )
try: response = await handler.retrieve_responses(
response = await handler.retrieve_responses( response_id,
response_id, starting_after=starting_after,
starting_after=starting_after, stream=stream,
stream=stream, )
)
except Exception as e:
response = handler.create_error_response(e)
if isinstance(response, ErrorResponse): if isinstance(response, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -125,10 +120,7 @@ async def cancel_responses(response_id: str, raw_request: Request):
message="The model does not support Responses API" message="The model does not support Responses API"
) )
try: response = await handler.cancel_responses(response_id)
response = await handler.cancel_responses(response_id)
except Exception as e:
response = handler.create_error_response(e)
if isinstance(response, ErrorResponse): if isinstance(response, ErrorResponse):
return JSONResponse( return JSONResponse(
+89 -126
View File
@@ -11,7 +11,6 @@ from copy import copy
from http import HTTPStatus from http import HTTPStatus
from typing import Final from typing import Final
import jinja2
from fastapi import Request from fastapi import Request
from openai.types.responses import ( from openai.types.responses import (
ResponseContentPartAddedEvent, ResponseContentPartAddedEvent,
@@ -174,14 +173,12 @@ class OpenAIServingResponses(OpenAIServing):
enable_prompt_tokens_details: bool = False, enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
enable_log_outputs: bool = False, enable_log_outputs: bool = False,
log_error_stack: bool = False,
) -> None: ) -> None:
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
log_error_stack=log_error_stack,
) )
self.chat_template = chat_template self.chat_template = chat_template
@@ -365,28 +362,15 @@ class OpenAIServingResponses(OpenAIServing):
else: else:
prev_response = None prev_response = None
try: lora_request = self._maybe_get_adapters(request)
lora_request = self._maybe_get_adapters(request) model_name = self.models.model_name(lora_request)
model_name = self.models.model_name(lora_request)
if self.use_harmony: if self.use_harmony:
messages, engine_prompts = self._make_request_with_harmony( messages, engine_prompts = self._make_request_with_harmony(
request, prev_response request, prev_response
) )
else: else:
messages, engine_prompts = await self._make_request( messages, engine_prompts = await self._make_request(request, prev_response)
request, prev_response
)
except (
ValueError,
TypeError,
RuntimeError,
jinja2.TemplateError,
NotImplementedError,
) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(e)
request_metadata = RequestResponseMetadata(request_id=request.request_id) request_metadata = RequestResponseMetadata(request_id=request.request_id)
if raw_request: if raw_request:
@@ -424,86 +408,83 @@ class OpenAIServingResponses(OpenAIServing):
else: else:
assert len(builtin_tool_list) == 0 assert len(builtin_tool_list) == 0
available_tools = [] available_tools = []
try: tokenizer = self.renderer.get_tokenizer()
tokenizer = self.renderer.get_tokenizer()
for engine_prompt in engine_prompts: for engine_prompt in engine_prompts:
maybe_error = self._validate_generator_input(engine_prompt) maybe_error = self._validate_generator_input(engine_prompt)
if maybe_error is not None: if maybe_error is not None:
return maybe_error return maybe_error
default_max_tokens = get_max_tokens( default_max_tokens = get_max_tokens(
max_model_len, max_model_len,
request.max_output_tokens, request.max_output_tokens,
self._extract_prompt_len(engine_prompt), self._extract_prompt_len(engine_prompt),
self.default_sampling_params, self.default_sampling_params,
self.override_max_tokens, self.override_max_tokens,
) )
sampling_params = request.to_sampling_params( sampling_params = request.to_sampling_params(
default_max_tokens, self.default_sampling_params default_max_tokens, self.default_sampling_params
) )
trace_headers = ( trace_headers = (
None None
if raw_request is None if raw_request is None
else await self._get_trace_headers(raw_request.headers) else await self._get_trace_headers(raw_request.headers)
) )
context: ConversationContext context: ConversationContext
if self.use_harmony: if self.use_harmony:
if request.stream: if request.stream:
context = StreamingHarmonyContext(messages, available_tools) context = StreamingHarmonyContext(messages, available_tools)
else:
context = HarmonyContext(messages, available_tools)
else: else:
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: context = HarmonyContext(messages, available_tools)
# This is a feature in development for parsing else:
# tokens during generation instead of at the end if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
context = ParsableContext( # This is a feature in development for parsing
response_messages=messages, # tokens during generation instead of at the end
tokenizer=tokenizer, context = ParsableContext(
reasoning_parser_cls=self.parser.reasoning_parser_cls response_messages=messages,
if self.parser tokenizer=tokenizer,
else None, reasoning_parser_cls=self.parser.reasoning_parser_cls
request=request, if self.parser
tool_parser_cls=self.parser.tool_parser_cls else None,
if self.parser request=request,
else None, tool_parser_cls=self.parser.tool_parser_cls
available_tools=available_tools, if self.parser
chat_template=self.chat_template, else None,
chat_template_content_format=self.chat_template_content_format, available_tools=available_tools,
) chat_template=self.chat_template,
else: chat_template_content_format=self.chat_template_content_format,
context = SimpleContext() )
else:
context = SimpleContext()
if self.parser and self.parser.reasoning_parser_cls is not None: if self.parser and self.parser.reasoning_parser_cls is not None:
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer) reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
if ( if (
isinstance( isinstance(
struct_out := sampling_params.structured_outputs, struct_out := sampling_params.structured_outputs,
StructuredOutputsParams, StructuredOutputsParams,
) )
and struct_out.all_non_structural_tag_constraints_none() and struct_out.all_non_structural_tag_constraints_none()
): ):
sampling_params.structured_outputs = replace( sampling_params.structured_outputs = replace(
struct_out, struct_out,
structural_tag=reasoning_parser.prepare_structured_tag( structural_tag=reasoning_parser.prepare_structured_tag(
struct_out.structural_tag, self.tool_server struct_out.structural_tag, self.tool_server
), ),
) )
generator = self._generate_with_builtin_tools( generator = self._generate_with_builtin_tools(
request_id=request.request_id, request_id=request.request_id,
engine_prompt=engine_prompt, engine_prompt=engine_prompt,
sampling_params=sampling_params, sampling_params=sampling_params,
context=context, context=context,
lora_request=lora_request, lora_request=lora_request,
priority=request.priority, priority=request.priority,
trace_headers=trace_headers, trace_headers=trace_headers,
) )
generators.append(generator) generators.append(generator)
except ValueError as e:
return self.create_error_response(e)
assert len(generators) == 1 assert len(generators) == 1
(result_generator,) = generators (result_generator,) = generators
@@ -578,20 +559,15 @@ class OpenAIServingResponses(OpenAIServing):
request_metadata, request_metadata,
) )
try: return await self.responses_full_generator(
return await self.responses_full_generator( request,
request, sampling_params,
sampling_params, result_generator,
result_generator, context,
context, model_name,
model_name, tokenizer,
tokenizer, request_metadata,
request_metadata, )
)
except GenerationError as e:
return self._convert_generation_error_to_response(e)
except Exception as e:
return self.create_error_response(e)
async def _make_request( async def _make_request(
self, self,
@@ -675,8 +651,6 @@ class OpenAIServingResponses(OpenAIServing):
pass pass
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
# NOTE: Implementation of status is still WIP, but for now # NOTE: Implementation of status is still WIP, but for now
# we guarantee that if the status is not "completed", it is accurate. # we guarantee that if the status is not "completed", it is accurate.
@@ -1129,16 +1103,11 @@ class OpenAIServingResponses(OpenAIServing):
new_event_signal = asyncio.Event() new_event_signal = asyncio.Event()
self.event_store[request.request_id] = (event_deque, new_event_signal) self.event_store[request.request_id] = (event_deque, new_event_signal)
response = None response = None
generator = self.responses_stream_generator(request, *args, **kwargs)
try: try:
generator = self.responses_stream_generator(request, *args, **kwargs)
async for event in generator: async for event in generator:
event_deque.append(event) event_deque.append(event)
new_event_signal.set() # Signal new event available new_event_signal.set() # Signal new event available
except GenerationError as e:
response = self._convert_generation_error_to_response(e)
except Exception as e:
logger.exception("Background request failed for %s", request.request_id)
response = self.create_error_response(e)
finally: finally:
new_event_signal.set() new_event_signal.set()
@@ -1157,13 +1126,7 @@ class OpenAIServingResponses(OpenAIServing):
*args, *args,
**kwargs, **kwargs,
): ):
try: response = await self.responses_full_generator(request, *args, **kwargs)
response = await self.responses_full_generator(request, *args, **kwargs)
except GenerationError as e:
response = self._convert_generation_error_to_response(e)
except Exception as e:
logger.exception("Background request failed for %s", request.request_id)
response = self.create_error_response(e)
if isinstance(response, ErrorResponse): if isinstance(response, ErrorResponse):
# If the request has failed, update the status to "failed". # If the request has failed, update the status to "failed".
+76 -4
View File
@@ -11,7 +11,7 @@ from contextlib import asynccontextmanager
from http import HTTPStatus from http import HTTPStatus
import pydantic import pydantic
from fastapi import FastAPI, HTTPException, Request from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.exceptions import RequestValidationError from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from starlette.concurrency import iterate_in_threadpool from starlette.concurrency import iterate_in_threadpool
@@ -20,11 +20,13 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
from vllm import envs from vllm import envs
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.launcher import terminate_if_errored
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.entrypoints.utils import sanitize_message from vllm.entrypoints.utils import create_error_response, sanitize_message
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.gc_utils import freeze_gc_heap from vllm.utils.gc_utils import freeze_gc_heap
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger("vllm.entrypoints.openai.server_utils") logger = init_logger("vllm.entrypoints.openai.server_utils")
@@ -309,7 +311,69 @@ async def log_response(request: Request, call_next):
return response return response
async def http_exception_handler(_: Request, exc: HTTPException): async def engine_error_handler(
req: Request, exc: EngineDeadError | EngineGenerateError
):
"""
VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
method. This error could be request specific (and therefore
recoverable - e.g. if there is an error in input processing).
EngineDeadError is raised by the background output_handler
method. This error is global and therefore not recoverable.
We register these @app.exception_handlers to return nice
responses to the end user if they occur and shut down if needed.
See https://fastapi.tiangolo.com/tutorial/handling-errors/
for more details on how exception handlers work.
If an exception is encountered in a StreamingResponse
generator, the exception is not raised, since we already sent
a 200 status. Rather, we send an error message as the next chunk.
Since the exception is not raised, this means that the server
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
if req.app.state.args.log_error_stack:
logger.exception(
"Engine Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
terminate_if_errored(
server=req.app.state.server,
engine=req.app.state.engine_client,
)
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
async def exception_handler(req: Request, exc: Exception):
if req.app.state.args.log_error_stack:
logger.exception(
"Exception caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
async def http_exception_handler(req: Request, exc: HTTPException):
if req.app.state.args.log_error_stack:
logger.exception(
"HTTPException caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
err = ErrorResponse( err = ErrorResponse(
error=ErrorInfo( error=ErrorInfo(
message=sanitize_message(exc.detail), message=sanitize_message(exc.detail),
@@ -320,7 +384,15 @@ async def http_exception_handler(_: Request, exc: HTTPException):
return JSONResponse(err.model_dump(), status_code=exc.status_code) return JSONResponse(err.model_dump(), status_code=exc.status_code)
async def validation_exception_handler(_: Request, exc: RequestValidationError): async def validation_exception_handler(req: Request, exc: RequestValidationError):
if req.app.state.args.log_error_stack:
logger.exception(
"RequestValidationError caught. Request id: %s",
req.state.request_metadata.request_id
if hasattr(req.state, "request_metadata")
else None,
)
param = None param = None
errors = exc.errors() errors = exc.errors()
for error in errors: for error in errors:
@@ -71,10 +71,9 @@ async def create_transcriptions(
) )
audio_data = await request.file.read() audio_data = await request.file.read()
try:
generator = await handler.create_transcription(audio_data, request, raw_request) generator = await handler.create_transcription(audio_data, request, raw_request)
except Exception as e:
return handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code content=generator.model_dump(), status_code=generator.error.code
@@ -108,10 +107,8 @@ async def create_translations(
) )
audio_data = await request.file.read() audio_data = await request.file.read()
try:
generator = await handler.create_translation(audio_data, request, raw_request) generator = await handler.create_translation(audio_data, request, raw_request)
except Exception as e:
return handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -140,7 +137,6 @@ def init_transcription_state(
engine_client, engine_client,
state.openai_serving_models, state.openai_serving_models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=args.log_error_stack,
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
) )
if "transcription" in supported_tasks if "transcription" in supported_tasks
@@ -151,7 +147,6 @@ def init_transcription_state(
engine_client, engine_client,
state.openai_serving_models, state.openai_serving_models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=args.log_error_stack,
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
) )
if "transcription" in supported_tasks if "transcription" in supported_tasks
@@ -40,7 +40,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
*, *,
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
): ):
super().__init__( super().__init__(
@@ -49,7 +48,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="transcribe", task_type="transcribe",
log_error_stack=log_error_stack,
enable_force_include_usage=enable_force_include_usage, enable_force_include_usage=enable_force_include_usage,
) )
@@ -113,7 +111,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
*, *,
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
): ):
super().__init__( super().__init__(
@@ -122,7 +119,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="translate", task_type="translate",
log_error_stack=log_error_stack,
enable_force_include_usage=enable_force_include_usage, enable_force_include_usage=enable_force_include_usage,
) )
@@ -97,7 +97,6 @@ class OpenAISpeechToText(OpenAIServing):
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
task_type: Literal["transcribe", "translate"] = "transcribe", task_type: Literal["transcribe", "translate"] = "transcribe",
log_error_stack: bool = False,
enable_force_include_usage: bool = False, enable_force_include_usage: bool = False,
): ):
super().__init__( super().__init__(
@@ -105,7 +104,6 @@ class OpenAISpeechToText(OpenAIServing):
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
log_error_stack=log_error_stack,
) )
self.default_sampling_params = self.model_config.get_diff_sampling_param() self.default_sampling_params = self.model_config.get_diff_sampling_param()
@@ -517,69 +515,61 @@ class OpenAISpeechToText(OpenAIServing):
if raw_request: if raw_request:
raw_request.state.request_metadata = request_metadata raw_request.state.request_metadata = request_metadata
try: lora_request = self._maybe_get_adapters(request)
lora_request = self._maybe_get_adapters(request)
engine_prompts, duration_s = await self._preprocess_speech_to_text( engine_prompts, duration_s = await self._preprocess_speech_to_text(
request=request, request=request,
audio_data=audio_data, audio_data=audio_data,
request_id=request_id, request_id=request_id,
) )
except ValueError as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(e)
# Schedule the request and get the result generator. # Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len max_model_len = self.model_config.max_model_len
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
try: # Unlike most decoder-only models, whisper generation length is not
# Unlike most decoder-only models, whisper generation length is not # constrained by the size of the input audio, which is mapped to a
# constrained by the size of the input audio, which is mapped to a # fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
# fixed-size log-mel-spectogram. Still, allow for fewer tokens to be # generated by respecting the extra completion tokens arg.
# generated by respecting the extra completion tokens arg. max_tokens = get_max_tokens(
max_tokens = get_max_tokens( max_model_len,
max_model_len, request.max_completion_tokens,
request.max_completion_tokens, 0,
0, self.default_sampling_params,
self.default_sampling_params, )
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params,
)
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
list_result_generator = []
for i, engine_prompt in enumerate(engine_prompts):
request_id_item = f"{request_id}_{i}"
self._log_inputs(
request_id_item,
engine_prompt,
params=sampling_params,
lora_request=lora_request,
) )
sampling_params = request.to_sampling_params( trace_headers = (
max_tokens, None
self.default_sampling_params, if raw_request is None
else await self._get_trace_headers(raw_request.headers)
) )
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
list_result_generator = [] generator = self.engine_client.generate(
for i, engine_prompt in enumerate(engine_prompts): engine_prompt,
request_id_item = f"{request_id}_{i}" sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
self._log_inputs( list_result_generator.append(generator)
request_id_item,
engine_prompt,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
generator = self.engine_client.generate(
engine_prompt,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
except ValueError as e:
return self.create_error_response(e)
if request.stream: if request.stream:
return stream_generator_method( return stream_generator_method(
@@ -663,8 +653,6 @@ class OpenAISpeechToText(OpenAIServing):
return final_response return final_response
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
async def _speech_to_text_stream_generator( async def _speech_to_text_stream_generator(
self, self,
-4
View File
@@ -72,7 +72,6 @@ def init_pooling_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
log_error_stack=args.log_error_stack,
) )
) )
if any(t in supported_tasks for t in POOLING_TASKS) if any(t in supported_tasks for t in POOLING_TASKS)
@@ -86,7 +85,6 @@ def init_pooling_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
log_error_stack=args.log_error_stack,
) )
if "embed" in supported_tasks if "embed" in supported_tasks
else None else None
@@ -99,7 +97,6 @@ def init_pooling_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
log_error_stack=args.log_error_stack,
) )
if "classify" in supported_tasks if "classify" in supported_tasks
else None else None
@@ -114,7 +111,6 @@ def init_pooling_state(
state.openai_serving_models, state.openai_serving_models,
request_logger=request_logger, request_logger=request_logger,
score_template=resolved_chat_template, score_template=resolved_chat_template,
log_error_stack=args.log_error_stack,
use_gpu_for_pooling_score=getattr(args, "use_gpu_for_pooling_score", False), use_gpu_for_pooling_score=getattr(args, "use_gpu_for_pooling_score", False),
) )
if any(t in supported_tasks for t in ("embed", "score", "token_embed")) if any(t in supported_tasks for t in ("embed", "score", "token_embed"))
+16 -26
View File
@@ -41,7 +41,6 @@ from vllm.tracing import (
from vllm.utils import random_uuid from vllm.utils import random_uuid
from vllm.utils.async_utils import merge_async_iterators from vllm.utils.async_utils import merge_async_iterators
from ...utils import create_error_response
from .io_processor import PoolingIOProcessor from .io_processor import PoolingIOProcessor
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest) PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
@@ -112,34 +111,25 @@ class PoolingServing:
request: AnyPoolingRequest, request: AnyPoolingRequest,
raw_request: Request, raw_request: Request,
) -> JSONResponse: ) -> JSONResponse:
try: model_name = self.models.model_name()
model_name = self.models.model_name() request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
request_id = (
f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
)
await self._check_model(request) await self._check_model(request)
ctx = PoolingServeContext( ctx = PoolingServeContext(
request=request, request=request,
raw_request=raw_request, raw_request=raw_request,
model_name=model_name, model_name=model_name,
request_id=request_id, request_id=request_id,
) )
self._validate_request(ctx) self._validate_request(ctx)
self._maybe_get_adapters(ctx) self._maybe_get_adapters(ctx)
await self._preprocess(ctx) await self._preprocess(ctx)
await self._prepare_generators(ctx) await self._prepare_generators(ctx)
await self._collect_batch(ctx) await self._collect_batch(ctx)
response = await self._build_response(ctx) response = await self._build_response(ctx)
return JSONResponse(content=response.model_dump()) return JSONResponse(content=response.model_dump())
except Exception as e:
error_response = create_error_response(e)
return JSONResponse(
content=error_response.model_dump(),
status_code=error_response.error.code,
)
async def _preprocess( async def _preprocess(
self, self,
+1 -4
View File
@@ -61,10 +61,7 @@ async def create_embedding(
message="The model does not support Embeddings API" message="The model does not support Embeddings API"
) )
try: generator = await handler.create_embedding(request, raw_request)
generator = await handler.create_embedding(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
+201 -218
View File
@@ -54,13 +54,11 @@ class OpenAIServingEmbedding(OpenAIServing):
chat_template: str | None, chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption, chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False, trust_request_chat_template: bool = False,
log_error_stack: bool = False,
) -> None: ) -> None:
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=log_error_stack,
) )
self.chat_template = chat_template self.chat_template = chat_template
@@ -75,38 +73,34 @@ class OpenAIServingEmbedding(OpenAIServing):
self, self,
ctx: EmbeddingServeContext, ctx: EmbeddingServeContext,
) -> ErrorResponse | None: ) -> ErrorResponse | None:
try: ctx.lora_request = self._maybe_get_adapters(ctx.request)
ctx.lora_request = self._maybe_get_adapters(ctx.request)
if isinstance(ctx.request, EmbeddingChatRequest): if isinstance(ctx.request, EmbeddingChatRequest):
error_check_ret = self._validate_chat_template( error_check_ret = self._validate_chat_template(
request_chat_template=ctx.request.chat_template, request_chat_template=ctx.request.chat_template,
chat_template_kwargs=ctx.request.chat_template_kwargs, chat_template_kwargs=ctx.request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template, trust_request_chat_template=self.trust_request_chat_template,
) )
if error_check_ret is not None: if error_check_ret is not None:
return error_check_ret return error_check_ret
_, ctx.engine_prompts = await self._preprocess_chat( _, ctx.engine_prompts = await self._preprocess_chat(
ctx.request, ctx.request,
ctx.request.messages, ctx.request.messages,
default_template=self.chat_template, default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format, default_template_content_format=self.chat_template_content_format,
default_template_kwargs=None, default_template_kwargs=None,
) )
elif isinstance(ctx.request, EmbeddingCompletionRequest): elif isinstance(ctx.request, EmbeddingCompletionRequest):
ctx.engine_prompts = await self._preprocess_completion( ctx.engine_prompts = await self._preprocess_completion(
ctx.request, ctx.request,
prompt_input=ctx.request.input, prompt_input=ctx.request.input,
prompt_embeds=None, prompt_embeds=None,
) )
else: else:
return self.create_error_response("Invalid classification request type") return self.create_error_response("Invalid classification request type")
return None return None
except (ValueError, TypeError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
def request_output_to_embed_json_response( def request_output_to_embed_json_response(
self, self,
@@ -397,51 +391,47 @@ class OpenAIServingEmbedding(OpenAIServing):
# Custom logic for chunked processing # Custom logic for chunked processing
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
try: trace_headers = (
trace_headers = ( None
None if ctx.raw_request is None
if ctx.raw_request is None else await self._get_trace_headers(ctx.raw_request.headers)
else await self._get_trace_headers(ctx.raw_request.headers) )
pooling_params = self._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
if ctx.engine_prompts is None:
return self.create_error_response("Engine prompts not available")
max_pos_embeddings = self._get_max_position_embeddings()
for i, engine_prompt in enumerate(ctx.engine_prompts):
# Check if this specific prompt needs chunked processing
if "prompt_token_ids" in engine_prompt:
prompt_token_ids = engine_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
if len(prompt_token_ids) > max_pos_embeddings:
# Use chunked processing for this prompt
chunk_generators = await self._process_chunked_request(
ctx,
prompt_token_ids,
pooling_params,
trace_headers,
i,
)
generators.extend(chunk_generators)
continue
# Normal processing for short prompts or non-token prompts
generator = await self._create_single_prompt_generator(
ctx, engine_prompt, pooling_params, trace_headers, i
) )
generators.append(generator)
pooling_params = self._create_pooling_params(ctx) ctx.result_generator = merge_async_iterators(*generators)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
if ctx.engine_prompts is None: return None
return self.create_error_response("Engine prompts not available")
max_pos_embeddings = self._get_max_position_embeddings()
for i, engine_prompt in enumerate(ctx.engine_prompts):
# Check if this specific prompt needs chunked processing
if "prompt_token_ids" in engine_prompt:
prompt_token_ids = engine_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
if len(prompt_token_ids) > max_pos_embeddings:
# Use chunked processing for this prompt
chunk_generators = await self._process_chunked_request(
ctx,
prompt_token_ids,
pooling_params,
trace_headers,
i,
)
generators.extend(chunk_generators)
continue
# Normal processing for short prompts or non-token prompts
generator = await self._create_single_prompt_generator(
ctx, engine_prompt, pooling_params, trace_headers, i
)
generators.append(generator)
ctx.result_generator = merge_async_iterators(*generators)
return None
except Exception as e:
return self.create_error_response(e)
async def _collect_batch( async def _collect_batch(
self, self,
@@ -454,164 +444,157 @@ class OpenAIServingEmbedding(OpenAIServing):
minimize memory usage. minimize memory usage.
For regular requests, collects results normally. For regular requests, collects results normally.
""" """
try: if ctx.engine_prompts is None:
if ctx.engine_prompts is None: return self.create_error_response("Engine prompts not available")
return self.create_error_response("Engine prompts not available")
# Check if we used chunked processing # Check if we used chunked processing
use_chunked = self._should_use_chunked_processing(ctx.request) use_chunked = self._should_use_chunked_processing(ctx.request)
if not use_chunked: if not use_chunked:
return await super()._collect_batch(ctx=ctx) return await super()._collect_batch(ctx=ctx)
if ctx.result_generator is None: if ctx.result_generator is None:
return self.create_error_response("Result generator not available") return self.create_error_response("Result generator not available")
# Online aggregation for chunked requests to # Online aggregation for chunked requests to
# minimize memory usage # minimize memory usage
# Track aggregation state for each prompt # Track aggregation state for each prompt
prompt_aggregators: dict[int, dict[str, Any]] = {} prompt_aggregators: dict[int, dict[str, Any]] = {}
short_prompts_results: dict[int, PoolingRequestOutput] = {} short_prompts_results: dict[int, PoolingRequestOutput] = {}
async for result_idx, result in ctx.result_generator: async for result_idx, result in ctx.result_generator:
if "-chunk-" in result.request_id: if "-chunk-" in result.request_id:
# Extract prompt_idx from chunked request_id # Extract prompt_idx from chunked request_id
parts = result.request_id.split("-") parts = result.request_id.split("-")
try: try:
prompt_idx = int(parts[parts.index("prompt") + 1]) prompt_idx = int(parts[parts.index("prompt") + 1])
except (ValueError, IndexError): except (ValueError, IndexError):
# Fallback: extract from result_idx if parsing fails # Fallback: extract from result_idx if parsing fails
prompt_idx = result_idx prompt_idx = result_idx
# Initialize aggregator for this prompt if needed # Initialize aggregator for this prompt if needed
if prompt_idx not in prompt_aggregators: if prompt_idx not in prompt_aggregators:
prompt_aggregators[prompt_idx] = { prompt_aggregators[prompt_idx] = {
"weighted_sum": None, "weighted_sum": None,
"total_weight": 0, "total_weight": 0,
"chunk_count": 0, "chunk_count": 0,
"request_id": result.request_id.split("-chunk-")[0], "request_id": result.request_id.split("-chunk-")[0],
} }
aggregator = prompt_aggregators[prompt_idx] aggregator = prompt_aggregators[prompt_idx]
# MEAN pooling with online weighted averaging # MEAN pooling with online weighted averaging
# Ensure result is PoolingRequestOutput # Ensure result is PoolingRequestOutput
# for embedding processing # for embedding processing
if not isinstance(result, PoolingRequestOutput): if not isinstance(result, PoolingRequestOutput):
return self.create_error_response(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
)
# Handle both PoolingOutput and
# EmbeddingOutput types
if hasattr(result.outputs, "data"):
# PoolingOutput case
embedding_data = result.outputs.data
elif hasattr(result.outputs, "embedding"):
# EmbeddingOutput case -
# convert embedding list to tensor
embedding_data = result.outputs.embedding
else:
return self.create_error_response(
f"Unsupported output type: {type(result.outputs).__name__}"
)
if not isinstance(embedding_data, torch.Tensor):
embedding_data = torch.tensor(
embedding_data, dtype=torch.float32
)
if result.prompt_token_ids is None:
return self.create_error_response(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
if aggregator["weighted_sum"] is None:
# First chunk
aggregator["weighted_sum"] = weighted_embedding
else:
# Accumulate
aggregator["weighted_sum"] += weighted_embedding
aggregator["total_weight"] += weight
aggregator["chunk_count"] += 1
else:
# Non-chunked result - extract prompt_idx from request_id
parts = result.request_id.split("-")
try:
# Last part should be prompt index
prompt_idx = int(parts[-1])
except (ValueError, IndexError):
prompt_idx = result_idx # Fallback to result_idx
short_prompts_results[prompt_idx] = result
# Finalize aggregated results
final_res_batch: list[PoolingRequestOutput] = []
num_prompts = len(ctx.engine_prompts)
for prompt_idx in range(num_prompts):
if prompt_idx in prompt_aggregators:
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator["weighted_sum"]
total_weight = aggregator["total_weight"]
if (
weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight, (int, float))
and total_weight > 0
):
# Compute final mean embedding
final_embedding = weighted_sum / total_weight
# Create a PoolingRequestOutput
# for the aggregated result
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = ctx.engine_prompts[prompt_idx]
if "prompt_token_ids" not in original_prompt:
return self.create_error_response(
f"Chunked prompt {prompt_idx} does not contain "
"token IDs"
)
original_token_ids = original_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
pooling_request_output = PoolingRequestOutput(
request_id=aggregator["request_id"],
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
finished=True,
)
final_res_batch.append(pooling_request_output)
else:
return self.create_error_response(
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
elif prompt_idx in short_prompts_results:
final_res_batch.append(short_prompts_results[prompt_idx])
else:
return self.create_error_response( return self.create_error_response(
f"Result not found for prompt {prompt_idx}" f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
) )
ctx.final_res_batch = final_res_batch # Handle both PoolingOutput and
# EmbeddingOutput types
if hasattr(result.outputs, "data"):
# PoolingOutput case
embedding_data = result.outputs.data
elif hasattr(result.outputs, "embedding"):
# EmbeddingOutput case -
# convert embedding list to tensor
embedding_data = result.outputs.embedding
else:
return self.create_error_response(
f"Unsupported output type: {type(result.outputs).__name__}"
)
return None if not isinstance(embedding_data, torch.Tensor):
embedding_data = torch.tensor(embedding_data, dtype=torch.float32)
except Exception as e: if result.prompt_token_ids is None:
return self.create_error_response(e) return self.create_error_response(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
if aggregator["weighted_sum"] is None:
# First chunk
aggregator["weighted_sum"] = weighted_embedding
else:
# Accumulate
aggregator["weighted_sum"] += weighted_embedding
aggregator["total_weight"] += weight
aggregator["chunk_count"] += 1
else:
# Non-chunked result - extract prompt_idx from request_id
parts = result.request_id.split("-")
try:
# Last part should be prompt index
prompt_idx = int(parts[-1])
except (ValueError, IndexError):
prompt_idx = result_idx # Fallback to result_idx
short_prompts_results[prompt_idx] = result
# Finalize aggregated results
final_res_batch: list[PoolingRequestOutput] = []
num_prompts = len(ctx.engine_prompts)
for prompt_idx in range(num_prompts):
if prompt_idx in prompt_aggregators:
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator["weighted_sum"]
total_weight = aggregator["total_weight"]
if (
weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight, (int, float))
and total_weight > 0
):
# Compute final mean embedding
final_embedding = weighted_sum / total_weight
# Create a PoolingRequestOutput
# for the aggregated result
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = ctx.engine_prompts[prompt_idx]
if "prompt_token_ids" not in original_prompt:
return self.create_error_response(
f"Chunked prompt {prompt_idx} does not contain token IDs"
)
original_token_ids = original_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
pooling_request_output = PoolingRequestOutput(
request_id=aggregator["request_id"],
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
finished=True,
)
final_res_batch.append(pooling_request_output)
else:
return self.create_error_response(
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
elif prompt_idx in short_prompts_results:
final_res_batch.append(short_prompts_results[prompt_idx])
else:
return self.create_error_response(
f"Result not found for prompt {prompt_idx}"
)
ctx.final_res_batch = final_res_batch
return None
async def create_embedding( async def create_embedding(
self, self,
@@ -41,10 +41,8 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
return base_server.create_error_response( return base_server.create_error_response(
message="The model does not support Pooling API" message="The model does not support Pooling API"
) )
try:
generator = await handler.create_pooling(request, raw_request) generator = await handler.create_pooling(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
+72 -86
View File
@@ -8,7 +8,6 @@ from collections.abc import AsyncGenerator, Callable, Sequence
from functools import partial from functools import partial
from typing import Final, Literal, cast from typing import Final, Literal, cast
import jinja2
from fastapi import Request from fastapi import Request
from typing_extensions import assert_never from typing_extensions import assert_never
@@ -53,13 +52,11 @@ class OpenAIServingPooling(OpenAIServing):
chat_template: str | None, chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption, chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False, trust_request_chat_template: bool = False,
log_error_stack: bool = False,
) -> None: ) -> None:
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=log_error_stack,
) )
self.chat_template = chat_template self.chat_template = chat_template
@@ -84,101 +81,92 @@ class OpenAIServingPooling(OpenAIServing):
request_id = f"pool-{self._base_request_id(raw_request)}" request_id = f"pool-{self._base_request_id(raw_request)}"
created_time = int(time.time()) created_time = int(time.time())
try: lora_request = self._maybe_get_adapters(request)
lora_request = self._maybe_get_adapters(request)
if getattr(request, "dimensions", None) is not None: if getattr(request, "dimensions", None) is not None:
return self.create_error_response( return self.create_error_response("dimensions is currently not supported")
"dimensions is currently not supported"
engine_prompts: Sequence[ProcessorInputs]
if use_io_processor := isinstance(request, IOProcessorRequest):
if self.io_processor is None:
raise ValueError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details."
) )
engine_prompts: Sequence[ProcessorInputs] validated_prompt = self.io_processor.parse_data(request.data)
if use_io_processor := isinstance(request, IOProcessorRequest):
if self.io_processor is None:
raise ValueError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details."
)
validated_prompt = self.io_processor.parse_data(request.data) raw_prompts = await self.io_processor.pre_process_async(
prompt=validated_prompt, request_id=request_id
)
engine_prompts = await self._preprocess_cmpl(
request,
prompt_to_seq(raw_prompts),
)
elif isinstance(request, PoolingChatRequest):
error_check_ret = self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
raw_prompts = await self.io_processor.pre_process_async( _, engine_prompts = await self._preprocess_chat(
prompt=validated_prompt, request_id=request_id request,
) request.messages,
engine_prompts = await self._preprocess_cmpl( default_template=self.chat_template,
request, default_template_content_format=self.chat_template_content_format,
prompt_to_seq(raw_prompts), default_template_kwargs=None,
) )
elif isinstance(request, PoolingChatRequest): elif isinstance(request, PoolingCompletionRequest):
error_check_ret = self._validate_chat_template( engine_prompts = await self._preprocess_completion(
request_chat_template=request.chat_template, request,
chat_template_kwargs=request.chat_template_kwargs, prompt_input=request.input,
trust_request_chat_template=self.trust_request_chat_template, prompt_embeds=None,
) )
if error_check_ret is not None: else:
return error_check_ret raise ValueError(f"Unsupported request of type {type(request)}")
_, engine_prompts = await self._preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=None,
)
elif isinstance(request, PoolingCompletionRequest):
engine_prompts = await self._preprocess_completion(
request,
prompt_input=request.input,
prompt_embeds=None,
)
else:
raise ValueError(f"Unsupported request of type {type(request)}")
except (ValueError, TypeError, jinja2.TemplateError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
# Schedule the request and get the result generator. # Schedule the request and get the result generator.
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
try: if use_io_processor:
if use_io_processor: assert self.io_processor is not None
assert self.io_processor is not None
pooling_params = self.io_processor.merge_pooling_params() pooling_params = self.io_processor.merge_pooling_params()
if pooling_params.task is None: if pooling_params.task is None:
pooling_params.task = "plugin" pooling_params.task = "plugin"
else: else:
pooling_params = request.to_pooling_params() # type: ignore pooling_params = request.to_pooling_params() # type: ignore
for i, engine_prompt in enumerate(engine_prompts): for i, engine_prompt in enumerate(engine_prompts):
request_id_item = f"{request_id}-{i}" request_id_item = f"{request_id}-{i}"
self._log_inputs( self._log_inputs(
request_id_item, request_id_item,
engine_prompt, engine_prompt,
params=pooling_params, params=pooling_params,
lora_request=lora_request, lora_request=lora_request,
) )
trace_headers = ( trace_headers = (
None None
if raw_request is None if raw_request is None
else await self._get_trace_headers(raw_request.headers) else await self._get_trace_headers(raw_request.headers)
) )
generator = self.engine_client.encode( generator = self.engine_client.encode(
engine_prompt, engine_prompt,
pooling_params, pooling_params,
request_id_item, request_id_item,
lora_request=lora_request, lora_request=lora_request,
trace_headers=trace_headers, trace_headers=trace_headers,
priority=request.priority, priority=request.priority,
) )
generators.append(generator) generators.append(generator)
except ValueError as e:
return self.create_error_response(e)
result_generator = merge_async_iterators(*generators) result_generator = merge_async_iterators(*generators)
@@ -233,8 +221,6 @@ class OpenAIServingPooling(OpenAIServing):
) )
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
return response return response
+3 -8
View File
@@ -49,10 +49,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
message="The model does not support Score API" message="The model does not support Score API"
) )
try: generator = await handler.create_score(request, raw_request)
generator = await handler.create_score(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -100,10 +97,8 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
return base_server.create_error_response( return base_server.create_error_response(
message="The model does not support Rerank (Score) API" message="The model does not support Rerank (Score) API"
) )
try:
generator = await handler.do_rerank(request, raw_request) generator = await handler.do_rerank(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
@@ -62,7 +62,6 @@ class ServingScores(OpenAIServing):
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=log_error_stack,
) )
self.score_template = score_template self.score_template = score_template
self.use_gpu_for_pooling_score = use_gpu_for_pooling_score self.use_gpu_for_pooling_score = use_gpu_for_pooling_score
@@ -518,8 +517,6 @@ class ServingScores(OpenAIServing):
) )
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
async def do_rerank( async def do_rerank(
self, request: RerankRequest, raw_request: Request | None = None self, request: RerankRequest, raw_request: Request | None = None
@@ -562,8 +559,6 @@ class ServingScores(OpenAIServing):
) )
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
def request_output_to_score_response( def request_output_to_score_response(
self, self,
+2 -4
View File
@@ -64,10 +64,8 @@ async def generate(request: GenerateRequest, raw_request: Request):
return tokenization(raw_request).create_error_response( return tokenization(raw_request).create_error_response(
message="The model does not support generate tokens API" message="The model does not support generate tokens API"
) )
try:
generator = await handler.serve_tokens(request, raw_request) generator = await handler.serve_tokens(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
+26 -37
View File
@@ -49,7 +49,6 @@ class ServingTokens(OpenAIServing):
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
force_no_detokenize: bool = False, force_no_detokenize: bool = False,
return_tokens_as_token_ids: bool = False, return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
enable_prompt_tokens_details: bool = False, enable_prompt_tokens_details: bool = False,
enable_log_outputs: bool = False, enable_log_outputs: bool = False,
): ):
@@ -58,7 +57,6 @@ class ServingTokens(OpenAIServing):
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids, return_tokens_as_token_ids=return_tokens_as_token_ids,
log_error_stack=log_error_stack,
) )
self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_log_outputs = enable_log_outputs self.enable_log_outputs = enable_log_outputs
@@ -108,45 +106,38 @@ class ServingTokens(OpenAIServing):
# Schedule the request and get the result generator. # Schedule the request and get the result generator.
result_generator: AsyncGenerator[RequestOutput, None] | None = None result_generator: AsyncGenerator[RequestOutput, None] | None = None
try: sampling_params = request.sampling_params
sampling_params = request.sampling_params if self.force_no_detokenize:
if self.force_no_detokenize: sampling_params.detokenize = False
sampling_params.detokenize = False
self._log_inputs( self._log_inputs(
request_id, request_id,
engine_prompt, engine_prompt,
params=sampling_params, params=sampling_params,
lora_request=lora_request, lora_request=lora_request,
) )
trace_headers = ( trace_headers = (
None None
if raw_request is None if raw_request is None
else await self._get_trace_headers(raw_request.headers) else await self._get_trace_headers(raw_request.headers)
) )
result_generator = self.engine_client.generate( result_generator = self.engine_client.generate(
engine_prompt, engine_prompt,
sampling_params, sampling_params,
request_id, request_id,
lora_request=lora_request, lora_request=lora_request,
trace_headers=trace_headers, trace_headers=trace_headers,
priority=request.priority, priority=request.priority,
) )
except ValueError as e:
return self.create_error_response(str(e))
# TODO(NickLucche): Implement streaming response # TODO(NickLucche): Implement streaming response
try: assert result_generator is not None
assert result_generator is not None return await self.serve_tokens_full_generator(
return await self.serve_tokens_full_generator( request, result_generator, request_id, model_name, request_metadata
request, result_generator, request_id, model_name, request_metadata )
)
except ValueError as e:
return self.create_error_response(str(e))
async def serve_tokens_full_generator( async def serve_tokens_full_generator(
self, self,
@@ -165,8 +156,6 @@ class ServingTokens(OpenAIServing):
final_res = res final_res = res
except asyncio.CancelledError: except asyncio.CancelledError:
return self.create_error_response("Client disconnected") return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(str(e))
assert final_res is not None assert final_res is not None
@@ -49,10 +49,7 @@ router = APIRouter()
async def tokenize(request: TokenizeRequest, raw_request: Request): async def tokenize(request: TokenizeRequest, raw_request: Request):
handler = tokenization(raw_request) handler = tokenization(raw_request)
try: generator = await handler.create_tokenize(request, raw_request)
generator = await handler.create_tokenize(request, raw_request)
except Exception as e:
generator = handler.create_error_response(e)
if isinstance(generator, ErrorResponse): if isinstance(generator, ErrorResponse):
return JSONResponse( return JSONResponse(
+31 -41
View File
@@ -3,7 +3,6 @@
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Final from typing import Any, Final
import jinja2
from fastapi import Request from fastapi import Request
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
@@ -37,13 +36,11 @@ class OpenAIServingTokenization(OpenAIServing):
chat_template: str | None, chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption, chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False, trust_request_chat_template: bool = False,
log_error_stack: bool = False,
) -> None: ) -> None:
super().__init__( super().__init__(
engine_client=engine_client, engine_client=engine_client,
models=models, models=models,
request_logger=request_logger, request_logger=request_logger,
log_error_stack=log_error_stack,
) )
self.chat_template = chat_template self.chat_template = chat_template
@@ -61,40 +58,36 @@ class OpenAIServingTokenization(OpenAIServing):
request_id = f"tokenize-{self._base_request_id(raw_request)}" request_id = f"tokenize-{self._base_request_id(raw_request)}"
try: lora_request = self._maybe_get_adapters(request)
lora_request = self._maybe_get_adapters(request)
if isinstance(request, TokenizeChatRequest): if isinstance(request, TokenizeChatRequest):
tool_dicts = ( tool_dicts = (
None None
if request.tools is None if request.tools is None
else [tool.model_dump() for tool in request.tools] else [tool.model_dump() for tool in request.tools]
) )
error_check_ret = self._validate_chat_template( error_check_ret = self._validate_chat_template(
request_chat_template=request.chat_template, request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs, chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template, trust_request_chat_template=self.trust_request_chat_template,
) )
if error_check_ret is not None: if error_check_ret is not None:
return error_check_ret return error_check_ret
_, engine_prompts = await self._preprocess_chat( _, engine_prompts = await self._preprocess_chat(
request, request,
request.messages, request.messages,
default_template=self.chat_template, default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format, default_template_content_format=self.chat_template_content_format,
default_template_kwargs=None, default_template_kwargs=None,
tool_dicts=tool_dicts, tool_dicts=tool_dicts,
) )
else: else:
engine_prompts = await self._preprocess_completion( engine_prompts = await self._preprocess_completion(
request, request,
prompt_input=request.prompt, prompt_input=request.prompt,
prompt_embeds=None, prompt_embeds=None,
) )
except (ValueError, TypeError, jinja2.TemplateError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(f"{e} {e.__cause__}")
input_ids: list[int] = [] input_ids: list[int] = []
for engine_prompt in engine_prompts: for engine_prompt in engine_prompts:
@@ -152,12 +145,9 @@ class OpenAIServingTokenization(OpenAIServing):
self, self,
) -> TokenizerInfoResponse | ErrorResponse: ) -> TokenizerInfoResponse | ErrorResponse:
"""Get comprehensive tokenizer information.""" """Get comprehensive tokenizer information."""
try: tokenizer = self.renderer.get_tokenizer()
tokenizer = self.renderer.get_tokenizer() info = TokenizerInfo(tokenizer, self.chat_template).to_dict()
info = TokenizerInfo(tokenizer, self.chat_template).to_dict() return TokenizerInfoResponse(**info)
return TokenizerInfoResponse(**info)
except Exception as e:
return self.create_error_response(f"Failed to get tokenizer info: {str(e)}")
@dataclass @dataclass
+15 -29
View File
@@ -5,13 +5,10 @@ import asyncio
import dataclasses import dataclasses
import functools import functools
import os import os
import sys
import traceback
from argparse import Namespace from argparse import Namespace
from http import HTTPStatus from http import HTTPStatus
from logging import Logger from logging import Logger
from string import Template from string import Template
from typing import TYPE_CHECKING
import regex as re import regex as re
from fastapi import Request from fastapi import Request
@@ -20,24 +17,17 @@ from starlette.background import BackgroundTask, BackgroundTasks
from vllm import envs from vllm import envs
from vllm.engine.arg_utils import EngineArgs from vllm.engine.arg_utils import EngineArgs
from vllm.exceptions import VLLMValidationError from vllm.entrypoints.openai.engine.protocol import (
ErrorInfo,
ErrorResponse,
GenerationError,
StreamOptions,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import current_formatter_type, init_logger from vllm.logger import current_formatter_type, init_logger
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.argparse_utils import FlexibleArgumentParser
if TYPE_CHECKING:
from vllm.entrypoints.openai.engine.protocol import (
ErrorInfo,
ErrorResponse,
StreamOptions,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
else:
ErrorResponse = object
ErrorInfo = object
LoRAModulePath = object
StreamOptions = object
logger = init_logger(__name__) logger = init_logger(__name__)
VLLM_SUBCMD_PARSER_EPILOG = ( VLLM_SUBCMD_PARSER_EPILOG = (
@@ -307,20 +297,19 @@ def create_error_response(
err_type: str = "BadRequestError", err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
param: str | None = None, param: str | None = None,
log_error_stack: bool = False, ) -> ErrorResponse:
) -> "ErrorResponse":
exc: Exception | None = None exc: Exception | None = None
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
if isinstance(message, Exception): if isinstance(message, Exception):
exc = message exc = message
from vllm.exceptions import VLLMValidationError
if isinstance(exc, VLLMValidationError): if isinstance(exc, VLLMValidationError):
err_type = "BadRequestError" err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST status_code = HTTPStatus.BAD_REQUEST
param = exc.parameter param = exc.parameter
elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)): elif isinstance(exc, (ValueError, TypeError, OverflowError)):
# Common validation errors from user input # Common validation errors from user input
err_type = "BadRequestError" err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST status_code = HTTPStatus.BAD_REQUEST
@@ -329,6 +318,10 @@ def create_error_response(
err_type = "NotImplementedError" err_type = "NotImplementedError"
status_code = HTTPStatus.NOT_IMPLEMENTED status_code = HTTPStatus.NOT_IMPLEMENTED
param = None param = None
elif isinstance(exc, GenerationError):
err_type = "InternalServerError"
status_code = exc.status_code
param = None
elif exc.__class__.__name__ == "TemplateError": elif exc.__class__.__name__ == "TemplateError":
# jinja2.TemplateError (avoid importing jinja2) # jinja2.TemplateError (avoid importing jinja2)
err_type = "BadRequestError" err_type = "BadRequestError"
@@ -341,13 +334,6 @@ def create_error_response(
message = str(exc) message = str(exc)
if log_error_stack:
exc_type, _, _ = sys.exc_info()
if exc_type is not None:
traceback.print_exc()
else:
traceback.print_stack()
return ErrorResponse( return ErrorResponse(
error=ErrorInfo( error=ErrorInfo(
message=sanitize_message(message), message=sanitize_message(message),