[Frontend][Core] Standardize request error handling with VLLMError hierarchy (#49665)

Signed-off-by: Zach Zhu <zzqshu@126.com>
This commit is contained in:
Zach Zhu
2026-07-29 04:20:40 +00:00
committed by GitHub
parent f37f03db4a
commit 58f9659397
46 changed files with 370 additions and 248 deletions
+5 -1
View File
@@ -3,6 +3,8 @@
import pytest
from vllm.exceptions import VLLMValidationError
from ..conftest import IMAGE_ASSETS
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
@@ -19,7 +21,9 @@ models = ["llava-hf/llava-1.5-7b-hf"]
def test_context_length_too_short(vllm_runner, image_assets, model):
images = [asset.pil_image for asset in image_assets]
with pytest.raises(ValueError, match="longer than the maximum model length"):
with pytest.raises(
VLLMValidationError, match="longer than the maximum model length"
):
vllm_model = vllm_runner(
model,
# LLaVA has a feature size of 576
+2 -1
View File
@@ -6,6 +6,7 @@ import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
@@ -157,7 +158,7 @@ def test_chat_batch_failure_cleanup(llm_for_failure_test):
batch_2 = [valid_msg, valid_msg]
sampling_params = SamplingParams(temperature=0, max_tokens=10)
with pytest.raises(ValueError, match="maximum context length is"):
with pytest.raises(VLLMValidationError, match="maximum context length is"):
llm.chat(batch_1, sampling_params=sampling_params)
assert llm.llm_engine.get_num_unfinished_requests() == 0
@@ -5,17 +5,18 @@ import pytest
import torch
from vllm import LLM
from vllm.exceptions import VLLMValidationError
def test_empty_prompt():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(ValueError, match="decoder prompt cannot be empty"):
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
llm.generate([""])
def test_out_of_vocab_token():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(ValueError, match="out of vocabulary"):
with pytest.raises(VLLMValidationError, match="out of vocabulary"):
llm.generate({"prompt_token_ids": [999999]})
@@ -6,6 +6,7 @@ import pytest
from tests.entrypoints.multimodal.conftest import managed_llm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError
MODEL = "llava-hf/llava-1.5-7b-hf"
PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
@@ -42,7 +43,7 @@ def test_generate_with_embedding(llm: LLM):
def test_raw_image_rejected(llm: LLM):
"""Raw image input is still rejected when limit=0."""
raw_image = ImageAsset("stop_sign").pil_image
with pytest.raises(ValueError, match=r"At most 0 image\(s\)"):
with pytest.raises(VLLMValidationError, match=r"At most 0 image\(s\)"):
llm.generate(
{"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
sampling_params=SamplingParams(max_tokens=16),
@@ -18,6 +18,7 @@ from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
# any model with a chat template should work here
@@ -1074,7 +1075,7 @@ def test_chat_completion_request_n_parameter_exceeds_default_limit(
max_tokens=10,
)
with pytest.raises(ValueError, match="n must be at most"):
with pytest.raises(VLLMValidationError, match="n must be at most"):
request.to_sampling_params(
max_tokens=10,
default_sampling_params={},
@@ -1136,7 +1137,7 @@ def test_chat_completion_request_n_parameter_custom_limit(
max_tokens=10,
)
with pytest.raises(ValueError, match="n must be at most 128"):
with pytest.raises(VLLMValidationError, match="n must be at most 128"):
request_over.to_sampling_params(
max_tokens=10,
default_sampling_params={},
@@ -1160,7 +1161,7 @@ def test_chat_completion_request_n_parameter_massive_value(
max_tokens=1,
)
with pytest.raises(ValueError, match="n must be at most"):
with pytest.raises(VLLMValidationError, match="n must be at most"):
request.to_sampling_params(
max_tokens=1,
default_sampling_params={},
@@ -6,7 +6,6 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -18,6 +17,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.exceptions import VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -479,7 +479,7 @@ def test_json_schema_response_format_missing_schema():
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
with pytest.raises(
ValidationError,
VLLMValidationError,
match="Invalid response_format structural_tag",
):
ChatCompletionRequest(
@@ -493,7 +493,7 @@ def test_structural_tag_response_format_invalid(format_value):
def test_batch_structural_tag_response_format_invalid(format_value):
"""Batch chat should reject malformed structural tags at request parsing."""
with pytest.raises(
ValidationError,
VLLMValidationError,
match="Invalid response_format structural_tag",
):
BatchChatCompletionRequest(
@@ -507,7 +507,7 @@ def test_batch_structural_tag_response_format_invalid(format_value):
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
ValidationError,
VLLMValidationError,
match="Invalid structured_outputs structural_tag",
):
ChatCompletionRequest(
@@ -521,7 +521,7 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
with pytest.raises(ValidationError, match=f"`{field_name}` must be an integer"):
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
@@ -14,11 +14,11 @@ digit-token vocab id).
import math
import pytest
from pydantic import ValidationError
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.exceptions import VLLMValidationError
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -87,7 +87,7 @@ def test_completion_request_decouples_top_k_from_explicit_token_ids():
def test_completion_rejects_explicit_token_ids_without_generated_tokens():
with pytest.raises(ValidationError, match="no output tokens are generated"):
with pytest.raises(VLLMValidationError, match="no output tokens are generated"):
CompletionRequest(
model=MODEL_NAME,
prompt="Hello",
@@ -99,7 +99,7 @@ def test_completion_rejects_explicit_token_ids_without_generated_tokens():
def test_requests_reject_explicit_token_ids_with_beam_search():
with pytest.raises(ValidationError, match="not supported with beam search"):
with pytest.raises(VLLMValidationError, match="not supported with beam search"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Hello"}],
@@ -108,7 +108,7 @@ def test_requests_reject_explicit_token_ids_with_beam_search():
use_beam_search=True,
)
with pytest.raises(ValidationError, match="not supported with beam search"):
with pytest.raises(VLLMValidationError, match="not supported with beam search"):
CompletionRequest(
model=MODEL_NAME,
prompt="Hello",
@@ -2,15 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from pydantic import ValidationError
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.exceptions import VLLMValidationError
@pytest.mark.parametrize("raw_value", [-2, 0.6, 10.5])
def test_chat_completion_request_rejects_invalid_thinking_token_budget(raw_value):
with pytest.raises(ValidationError, match="thinking_token_budget"):
with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
ChatCompletionRequest.model_validate(
{
"model": "qwen",
@@ -44,7 +44,7 @@ def test_chat_completion_request_accepts_minus_one_as_unlimited():
@pytest.mark.parametrize("raw_value", [0.6, 3.14, -2])
def test_completion_request_rejects_invalid_thinking_token_budget(raw_value):
with pytest.raises(ValidationError, match="thinking_token_budget"):
with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
CompletionRequest.model_validate(
{
"model": "qwen",
@@ -6,7 +6,6 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import ValidationError
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -18,6 +17,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.exceptions import VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -430,7 +430,7 @@ def test_json_schema_response_format_missing_schema():
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
with pytest.raises(
ValidationError,
VLLMValidationError,
match="Invalid response_format structural_tag",
):
CompletionRequest(
@@ -445,7 +445,7 @@ def test_structural_tag_response_format_invalid(format_value):
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
ValidationError,
VLLMValidationError,
match="Invalid structured_outputs structural_tag",
):
CompletionRequest(
@@ -616,7 +616,7 @@ class TestCompletionPromptListLimit:
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
with pytest.raises(ValidationError, match=f"`{field_name}` must be an integer"):
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
@@ -13,6 +13,7 @@ import torch
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.renderers.embed_utils import safe_load_prompt_embeds
@@ -111,5 +112,5 @@ def test_disable_prompt_embeds(dtype: torch.dtype, seq_len: int, hidden_size: in
buffer.seek(0)
encoded_tensor = pybase64.b64encode(buffer.getvalue())
with pytest.raises(ValueError, match="--enable-prompt-embeds"):
with pytest.raises(VLLMValidationError, match="--enable-prompt-embeds"):
safe_load_prompt_embeds(model_config, encoded_tensor)
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
ResponseTextConfig,
)
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import StructuredOutputsParams
@@ -163,7 +164,7 @@ class TestResponsesRequestSamplingParams:
text=text_config,
)
with pytest.raises(ValueError) as exc_info:
with pytest.raises(VLLMValidationError) as exc_info:
request.to_sampling_params(default_max_tokens=1000)
assert "Cannot specify both structured_outputs and text.format" in str(
@@ -8,54 +8,95 @@ PrometheusInstrumentatorMiddleware before being caught by ServerErrorMiddleware.
"""
from argparse import Namespace
from http import HTTPStatus
import httpx
import pytest
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi import HTTPException
from prometheus_client import CollectorRegistry
from prometheus_fastapi_instrumentator import Instrumentator
from vllm.entrypoints.serve.utils.server_utils import exception_handler
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
from vllm.entrypoints.openai.api_server import build_app
from vllm.exceptions import (
VLLMNotFoundError,
VLLMServerError,
VLLMValidationError,
)
@pytest.fixture
@pytest.fixture(scope="module")
def should_do_global_cleanup_after_test() -> bool:
# This suite never initializes distributed/accelerator state.
return False
def _build_args() -> Namespace:
"""Minimal args for ``build_app``; avoids ``make_arg_parser`` device probing."""
return Namespace(
disable_fastapi_docs=True,
enable_offline_docs=False,
root_path=None,
allowed_origins=["*"],
allow_credentials=False,
allowed_methods=["*"],
allowed_headers=["*"],
api_key=None,
enable_request_id_headers=False,
enable_fault_tolerance=False,
middleware=[],
log_error_stack=False,
)
@pytest.fixture(scope="module")
def registry():
"""Create a fresh Prometheus registry for each test."""
"""Shared Prometheus registry for the module-scoped app."""
return CollectorRegistry()
@pytest.fixture
@pytest.fixture(scope="module")
def app(registry):
"""Create a minimal FastAPI app that mirrors vLLM's exception handler
and Prometheus middleware setup."""
"""Build the real vLLM FastAPI app once and attach probe routes that raise.
app = FastAPI()
Patch the name used by ``attach_router`` (imported into the instrumentator
metrics module), not ``vllm.v1.metrics.prometheus`` alone — that binding is
captured at import time.
"""
import vllm.entrypoints.serve.instrumentator.metrics as metrics_mod
# Mock app state that exception_handler needs
app.state.args = Namespace(log_error_stack=False)
original = metrics_mod.get_prometheus_registry
metrics_mod.get_prometheus_registry = lambda: registry
try:
app = build_app(_build_args(), supported_tasks=())
finally:
metrics_mod.get_prometheus_registry = original
# Register exception handlers exactly as vLLM does in build_app()
app.exception_handler(HTTPException)(_http_exception_handler)
app.exception_handler(RequestValidationError)(_validation_exception_handler)
app.exception_handler(ValueError)(exception_handler)
app.exception_handler(TypeError)(exception_handler)
app.exception_handler(OverflowError)(exception_handler)
app.exception_handler(NotImplementedError)(exception_handler)
app.exception_handler(VLLMValidationError)(exception_handler)
app.exception_handler(VLLMNotFoundError)(exception_handler)
app.exception_handler(Exception)(exception_handler)
@app.get("/raise_http_exception_400")
async def raise_http_exception_400():
raise HTTPException(status_code=400, detail="bad request")
# Instrument with Prometheus (same as vLLM's attach_router)
Instrumentator(
excluded_handlers=["/metrics"],
registry=registry,
).add().instrument(app)
@app.get("/raise_http_exception_404")
async def raise_http_exception_404():
raise HTTPException(status_code=404, detail="not found")
@app.get("/raise_request_validation_error")
async def raise_request_validation_error(n: int):
# Invalid ``n`` triggers FastAPI's RequestValidationError.
return {"n": n}
@app.get("/raise_vllm_validation_error")
async def raise_vllm_validation_error():
raise VLLMValidationError("bad parameter", parameter="temperature")
@app.get("/raise_vllm_not_found_error")
async def raise_vllm_not_found_error():
raise VLLMNotFoundError("model not found")
@app.get("/raise_vllm_server_error")
async def raise_vllm_server_error():
# Bare VLLMServerError goes through vllm_error_handler → 500.
# EngineGenerateError / EngineDeadError are not used here: they call
# terminate_if_errored and need engine/server state.
raise VLLMServerError("internal server failure")
# Test routes that raise different exception types
@app.get("/raise_value_error")
async def raise_value_error():
raise ValueError("invalid input value")
@@ -72,22 +113,6 @@ def app(registry):
async def raise_not_implemented_error():
raise NotImplementedError("feature not supported")
@app.get("/raise_vllm_validation_error")
async def raise_vllm_validation_error():
raise VLLMValidationError("bad parameter", parameter="temperature")
@app.get("/raise_vllm_not_found_error")
async def raise_vllm_not_found_error():
raise VLLMNotFoundError("model not found")
@app.get("/raise_http_exception_400")
async def raise_http_exception_400():
raise HTTPException(status_code=400, detail="bad request")
@app.get("/raise_http_exception_404")
async def raise_http_exception_404():
raise HTTPException(status_code=404, detail="not found")
@app.get("/raise_runtime_error")
async def raise_runtime_error():
raise RuntimeError("unexpected server error")
@@ -99,14 +124,6 @@ def app(registry):
return app
async def _http_exception_handler(req: Request, exc: HTTPException):
return JSONResponse({"error": exc.detail}, status_code=exc.status_code)
async def _validation_exception_handler(req: Request, exc: RequestValidationError):
return JSONResponse({"error": str(exc)}, status_code=HTTPStatus.BAD_REQUEST)
def _get_http_requests_total(registry, method: str, handler: str):
"""Extract the http_requests_total metric values grouped by status.
@@ -128,31 +145,31 @@ def _get_http_requests_total(registry, method: str, handler: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"endpoint,expected_status_group,expected_http_code",
"endpoint,expected_status_group,expected_http_code,request_kwargs",
[
# These should record as 4xx in Prometheus
("/raise_value_error", "4xx", 400),
("/raise_type_error", "4xx", 400),
("/raise_overflow_error", "4xx", 400),
("/raise_vllm_validation_error", "4xx", 400),
("/raise_vllm_not_found_error", "4xx", 404),
("/raise_http_exception_400", "4xx", 400),
("/raise_http_exception_404", "4xx", 404),
# NotImplementedError returns 501 which is still 5xx group
("/raise_not_implemented_error", "5xx", 501),
# These should record as 5xx in Prometheus (genuine server errors)
("/raise_runtime_error", "5xx", 500),
# Successful requests should record as 2xx
("/success", "2xx", 200),
("/raise_http_exception_400", "4xx", 400, {}),
("/raise_http_exception_404", "4xx", 404, {}),
("/raise_request_validation_error", "4xx", 400, {"params": {"n": "x"}}),
("/raise_vllm_validation_error", "4xx", 400, {}),
("/raise_vllm_not_found_error", "4xx", 404, {}),
("/raise_vllm_server_error", "5xx", 500, {}),
("/raise_value_error", "4xx", 400, {}),
("/raise_type_error", "4xx", 400, {}),
("/raise_overflow_error", "4xx", 400, {}),
("/raise_not_implemented_error", "5xx", 501, {}),
("/raise_runtime_error", "5xx", 500, {}),
("/success", "2xx", 200, {}),
],
ids=[
"HTTPException(400)->4xx",
"HTTPException(404)->4xx",
"RequestValidationError->4xx",
"VLLMValidationError->4xx",
"VLLMNotFoundError->4xx",
"VLLMServerError->5xx",
"ValueError->4xx",
"TypeError->4xx",
"OverflowError->4xx",
"VLLMValidationError->4xx",
"VLLMNotFoundError->4xx",
"HTTPException(400)->4xx",
"HTTPException(404)->4xx",
"NotImplementedError->5xx",
"RuntimeError->5xx",
"success->2xx",
@@ -164,6 +181,7 @@ async def test_http_requests_total_records_correct_status(
endpoint,
expected_status_group,
expected_http_code,
request_kwargs,
):
"""Verify that http_requests_total records the correct status group.
@@ -177,7 +195,7 @@ async def test_http_requests_total_records_correct_status(
async with httpx.AsyncClient(
transport=transport, base_url="http://testserver"
) as client:
response = await client.get(endpoint)
response = await client.get(endpoint, **request_kwargs)
# Verify the HTTP response code returned to the client is correct
assert response.status_code == expected_http_code, (
@@ -21,6 +21,7 @@ from vllm.entrypoints.chat_utils import (
parse_chat_messages,
parse_chat_messages_async,
)
from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.multimodal.utils import (
encode_audio_url,
@@ -1504,7 +1505,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
"ignore",
message="coroutine 'async_get_and_parse_image' was never awaited",
)
with pytest.raises(ValueError, match="At most"):
with pytest.raises(VLLMValidationError, match="At most"):
parse_chat_messages(
[
{
@@ -1540,7 +1541,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
"ignore",
message="coroutine 'async_get_and_parse_image' was never awaited",
)
with pytest.raises(ValueError, match="At most"):
with pytest.raises(VLLMValidationError, match="At most"):
parse_chat_messages(
[
{
@@ -7,6 +7,7 @@ import pytest
import torch
from PIL import Image as PILImage
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
@@ -222,7 +223,7 @@ def test_limit_mm_per_prompt(
mm_data = {"image": images}
# Expect ValueError when exceeding limit
with pytest.raises(ValueError, match="At most 1 image"):
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
@@ -7,6 +7,7 @@ import pytest
import torch
from PIL import Image as PILImage
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
@@ -197,7 +198,7 @@ def test_limit_mm_per_prompt(
mm_data = {"image": images}
with pytest.raises(ValueError, match="At most 1 image"):
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
@@ -14,7 +14,7 @@ import aiohttp
import pytest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.exceptions import VLLMUnprocessableEntityError
from vllm.exceptions import VLLMClientError, VLLMUnprocessableEntityError
from vllm.multimodal.media import MediaConnector
@@ -35,9 +35,9 @@ class TestVLLMUnprocessableEntityError:
assert "parameter=image_url" in str(exc)
assert "value=https://example.com/image.jpg" in str(exc)
def test_is_value_error_subclass(self):
def test_is_client_error_subclass(self):
exc = VLLMUnprocessableEntityError("Test")
assert isinstance(exc, ValueError)
assert isinstance(exc, VLLMClientError)
class TestMediaConnectorErrorHandling:
+6 -1
View File
@@ -8,6 +8,7 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.multimodal.processing.processor import (
@@ -931,7 +932,11 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
else:
mm_data = {"image": [image] * num_images}
exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
exc_ctx = (
nullcontext()
if is_valid
else pytest.raises(VLLMValidationError, match="At most")
)
with exc_ctx:
processor(
@@ -26,6 +26,7 @@ from vllm.entrypoints.chat_utils import (
parse_chat_messages,
parse_chat_messages_async,
)
from vllm.exceptions import VLLMValidationError
from vllm.renderers.hf import (
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR,
_build_mixed_prompt_embeds,
@@ -264,7 +265,7 @@ def test_parse_chat_messages_requires_flag():
"content": [{"type": "prompt_embeds", "data": b64}],
}
]
with pytest.raises(ValueError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
with pytest.raises(VLLMValidationError, match=_ENABLE_PROMPT_EMBEDS_ERROR):
parse_chat_messages(
messages,
mc,
@@ -283,7 +284,7 @@ def test_parse_chat_messages_rejects_missing_data():
"content": [{"type": "prompt_embeds"}], # no `data`
}
]
with pytest.raises(ValueError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
with pytest.raises(VLLMValidationError, match=_PROMPT_EMBEDS_MISSING_DATA_ERROR):
parse_chat_messages(
messages,
mc,
+4 -3
View File
@@ -11,6 +11,7 @@ import pytest
import torch
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import SingletonPrompt
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import HfRenderer
@@ -286,7 +287,7 @@ class TestRenderPrompt:
)
with pytest.raises(
ValueError,
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
@@ -307,7 +308,7 @@ class TestRenderPrompt:
)
with pytest.raises(
ValueError,
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
@@ -328,7 +329,7 @@ class TestRenderPrompt:
)
with pytest.raises(
ValueError,
VLLMValidationError,
match="maximum context length is",
):
renderer.tokenize_prompts(
+1 -1
View File
@@ -42,7 +42,7 @@ class TestNonFiniteRepetitionPenalty:
ids=["nan", "inf", "-inf", "math.nan", "math.inf"],
)
def test_non_finite_repetition_penalty_rejected(self, value: float):
with pytest.raises(ValueError, match="repetition_penalty"):
with pytest.raises(VLLMValidationError, match="repetition_penalty"):
SamplingParams(repetition_penalty=value)
def test_finite_repetition_penalty_accepted(self):
+3 -2
View File
@@ -15,6 +15,7 @@ from vllm.envs import (
env_with_choices,
environment_variables,
)
from vllm.exceptions import VLLMValidationError
def test_getattr_without_cache(monkeypatch: pytest.MonkeyPatch):
@@ -547,7 +548,7 @@ class TestVllmMaxNSequences:
max_n = envs.VLLM_MAX_N_SEQUENCES
SamplingParams(n=max_n)
with pytest.raises(ValueError, match="n must be at most"):
with pytest.raises(VLLMValidationError, match="n must be at most"):
SamplingParams(n=max_n + 1)
def test_sampling_params_respects_custom_limit(
@@ -563,5 +564,5 @@ class TestVllmMaxNSequences:
SamplingParams(n=128)
with pytest.raises(ValueError, match="n must be at most 128"):
with pytest.raises(VLLMValidationError, match="n must be at most 128"):
SamplingParams(n=129)
+17 -11
View File
@@ -52,13 +52,19 @@ class MockModelConfig:
def test_removed_pooling_parameters(parameter: str, value: Any, message: str):
data = {"input": "hello", parameter: value}
for request_type in (EmbeddingRequest, ClassificationRequest, PoolingRequest):
with pytest.raises(ValidationError, match=message) as exc_info:
with pytest.raises(VLLMValidationError, match=message):
TypeAdapter(request_type).validate_python(data)
assert len(exc_info.value.errors()) == 1
with pytest.raises(ValidationError, match=message) as exc_info:
TypeAdapter(PoolerConfig).validate_python({parameter: value})
assert len(exc_info.value.errors()) == 1
# PoolerConfig still raises bare ValueError for `normalize`
# (wrapped to ValidationError by Pydantic), but `check_removed_pooling_task`
# raises VLLMValidationError for removed tasks.
if parameter == "normalize":
with pytest.raises(ValidationError, match=message) as exc_info:
TypeAdapter(PoolerConfig).validate_python({parameter: value})
assert len(exc_info.value.errors()) == 1
else:
with pytest.raises(VLLMValidationError, match=message):
TypeAdapter(PoolerConfig).validate_python({parameter: value})
if parameter == "task":
with pytest.raises(VLLMValidationError, match=message):
@@ -80,7 +86,7 @@ def test_embed():
invalid_parameters = classify_parameters + step_pooling_parameters
for p in set(invalid_parameters) - set(embed_parameters):
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, **{p: True})
pooling_params.verify(model_config)
@@ -100,7 +106,7 @@ def test_embed_dimensions(model_info: EmbedModelInfo):
pooling_params = PoolingParams(task=task, dimensions=None)
pooling_params.verify(model_config)
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, dimensions=1)
pooling_params.verify(model_config)
@@ -131,7 +137,7 @@ def test_embed_dimensions_matryoshka_without_list_upper_bound():
PoolingParams(task=task, dimensions=16).verify(model_config)
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
PoolingParams(task=task, dimensions=64).verify(model_config)
@@ -150,7 +156,7 @@ def test_classify(task):
invalid_parameters = embed_parameters + step_pooling_parameters
for p in set(invalid_parameters) - set(classify_parameters):
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, **{p: True})
pooling_params.verify(model_config)
@@ -176,7 +182,7 @@ def test_token_embed(pooling_type: str):
invalid_parameters = classify_parameters + step_pooling_parameters
for p in set(invalid_parameters) - set(embed_parameters):
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, **{p: True})
pooling_params.verify(model_config)
@@ -202,6 +208,6 @@ def test_token_classify(pooling_type: str):
invalid_parameters = embed_parameters + step_pooling_parameters
for p in set(invalid_parameters) - set(classify_parameters):
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
pooling_params = PoolingParams(task=task, **{p: True})
pooling_params.verify(model_config)
+2 -1
View File
@@ -5,6 +5,7 @@ from dataclasses import dataclass
import pytest
from vllm import SamplingParams
from vllm.exceptions import VLLMValidationError
@dataclass
@@ -32,7 +33,7 @@ class MockModelConfig:
)
def test_diffusion_rejects_unsupported_params(kwargs: dict):
params = SamplingParams(**kwargs)
with pytest.raises(ValueError, match="not yet supported with diffusion"):
with pytest.raises(VLLMValidationError, match="not yet supported with diffusion"):
params.verify(MockModelConfig(is_diffusion=True), None, None, None)
@@ -4,6 +4,7 @@
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.exceptions import VLLMValidationError
def test_chat_completion_request_with_no_tools():
@@ -27,7 +28,7 @@ def test_chat_completion_request_with_no_tools():
assert request.tool_choice == "none"
# tools key present but empty -- should be rejected
with pytest.raises(ValueError, match="must not be an empty array"):
with pytest.raises(VLLMValidationError, match="must not be an empty array"):
ChatCompletionRequest.model_validate(
{
"messages": [{"role": "user", "content": "Hello"}],
@@ -40,7 +41,7 @@ def test_chat_completion_request_with_no_tools():
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
def test_chat_completion_request_with_tool_choice_but_no_tools(tool_choice):
with pytest.raises(
ValueError, match="When using `tool_choice`, `tools` must be set."
VLLMValidationError, match="When using `tool_choice`, `tools` must be set."
):
ChatCompletionRequest.model_validate(
{
@@ -51,7 +52,7 @@ def test_chat_completion_request_with_tool_choice_but_no_tools(tool_choice):
)
with pytest.raises(
ValueError, match="When using `tool_choice`, `tools` must be set."
VLLMValidationError, match="When using `tool_choice`, `tools` must be set."
):
ChatCompletionRequest.model_validate(
{
@@ -134,7 +135,7 @@ SAMPLE_TOOL = {
def test_structured_outputs_with_named_tool_choice_rejected():
"""structured_outputs cannot be combined with a named tool_choice."""
with pytest.raises(
ValueError,
VLLMValidationError,
match="structured outputs or tools, not both",
):
ChatCompletionRequest.model_validate(
@@ -168,7 +169,7 @@ def test_structured_outputs_with_auto_tool_choice_allowed():
def test_multiple_structured_outputs_rejected():
"""Only one kind of structured output constraint is allowed."""
with pytest.raises(
ValueError,
VLLMValidationError,
match="You can only use one kind of constraints",
):
ChatCompletionRequest.model_validate(
@@ -2,12 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from pydantic import ValidationError
from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
ResponsesResponse,
)
from vllm.exceptions import VLLMValidationError
SAMPLE_TOOL = {
"type": "function",
@@ -58,13 +58,13 @@ def test_responses_request_required_without_tools(tools):
if tools is not None:
kwargs["tools"] = tools
with pytest.raises(
ValidationError, match="Tool choice 'required' must be specified"
VLLMValidationError, match="Tool choice 'required' must be specified"
):
ResponsesRequest.model_validate(kwargs)
def test_responses_request_named_tool_choice_without_tools():
with pytest.raises(ValidationError, match="not found in 'tools' parameter"):
with pytest.raises(VLLMValidationError, match="not found in 'tools' parameter"):
ResponsesRequest.model_validate(
{
"input": "Hello",
@@ -107,7 +107,7 @@ def test_responses_request_named_tool_choice_matching():
def test_responses_request_named_tool_choice_not_matching():
with pytest.raises(ValidationError, match="not found in 'tools' parameter"):
with pytest.raises(VLLMValidationError, match="not found in 'tools' parameter"):
ResponsesRequest.model_validate(
{
"input": "Hello",
@@ -164,7 +164,7 @@ def test_responses_request_empty_tools_tool_choice_auto():
],
)
def test_responses_request_named_tool_choice_missing_name(tool_choice):
with pytest.raises(ValidationError, match="not found in 'tools' parameter"):
with pytest.raises(VLLMValidationError, match="not found in 'tools' parameter"):
ResponsesRequest.model_validate(
{
"input": "Hello",
@@ -176,7 +176,7 @@ def test_responses_request_named_tool_choice_missing_name(tool_choice):
def test_responses_request_empty_tools_named_tool_choice():
with pytest.raises(ValidationError, match="not found in 'tools' parameter"):
with pytest.raises(VLLMValidationError, match="not found in 'tools' parameter"):
ResponsesRequest.model_validate(
{
"input": "Hello",
+1 -1
View File
@@ -59,7 +59,7 @@ def test_decoder_max_context_length_validation(
"Make sure that `max_model_len` is no smaller than the number of "
"text tokens (prompt + requested output tokens)."
)
with pytest.raises(ValueError) as excinfo:
with pytest.raises(VLLMValidationError) as excinfo:
vllm_model.generate_greedy(prompt_ids, max_tokens)
assert expected_msg in str(excinfo.value)
+3 -2
View File
@@ -16,6 +16,7 @@ Covers:
import pytest
from vllm import LLM, SamplingParams
from vllm.exceptions import VLLMValidationError
from vllm.outputs import RequestOutput
# Test configuration
@@ -479,13 +480,13 @@ def test_min_tokens_validation():
# Invalid cases
with pytest.raises(
ValueError,
VLLMValidationError,
match="min_tokens must be greater than or equal to 0",
):
SamplingParams(min_tokens=-1, max_tokens=10)
with pytest.raises(
ValueError,
VLLMValidationError,
match="min_tokens must be less than or equal to max_tokens",
):
SamplingParams(min_tokens=15, max_tokens=10)
+10 -3
View File
@@ -20,6 +20,7 @@ import pytest_asyncio
from vllm import SamplingParams
from vllm.engine.protocol import StreamingInput
from vllm.exceptions import VLLMValidationError
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
@@ -571,13 +572,17 @@ async def test_streaming_input_validation_errors(engine: AsyncLLM):
yield StreamingInput(prompt="test")
# Test n > 1 is rejected
with pytest.raises(ValueError, match="Input streaming not currently supported"):
with pytest.raises(
VLLMValidationError, match="Input streaming not currently supported"
):
params_n2 = SamplingParams(max_tokens=10, n=2)
async for _ in engine.generate(dummy_generator(), params_n2, "test_n2"):
pass
# Test FINAL_ONLY is rejected
with pytest.raises(ValueError, match="Input streaming not currently supported"):
with pytest.raises(
VLLMValidationError, match="Input streaming not currently supported"
):
params_final = SamplingParams(
max_tokens=10, output_kind=RequestOutputKind.FINAL_ONLY
)
@@ -585,7 +590,9 @@ async def test_streaming_input_validation_errors(engine: AsyncLLM):
pass
# Test stop strings are rejected
with pytest.raises(ValueError, match="Input streaming not currently supported"):
with pytest.raises(
VLLMValidationError, match="Input streaming not currently supported"
):
params_stop = SamplingParams(max_tokens=10, stop=["stop"])
async for _ in engine.generate(dummy_generator(), params_stop, "test_stop"):
pass
+4 -3
View File
@@ -19,6 +19,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import VLLMValidationError
from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
@@ -485,7 +486,7 @@ async def test_dp_rank_argument():
pass
# Test with out-of-range DP rank.
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
async for _ in engine.generate(
request_id="request-35",
prompt=TEXT_PROMPT,
@@ -554,8 +555,8 @@ async def test_header_dp_rank_argument():
# Test 2: Out-of-range DP rank (1)
mock_raw_request.headers = {"X-data-parallel-rank": "1"}
# should raise ValueError for out-of-range rank
with pytest.raises(ValueError):
# should raise VLLMValidationError for out-of-range rank
with pytest.raises(VLLMValidationError):
await serving_chat.create_chat_completion(req, mock_raw_request)
+2 -1
View File
@@ -11,6 +11,7 @@ import torch
from tests.utils import requires_spawn_multiprocessing
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor import (
@@ -61,7 +62,7 @@ class DummyLogitsProcessor(LogitsProcessor):
"target_token"
)
if target_token is not None and not isinstance(target_token, int):
raise ValueError(
raise VLLMValidationError(
f"target_token value {target_token} {type(target_token)} is not int"
)
+1 -1
View File
@@ -405,7 +405,7 @@ def test_max_logprobs():
runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
bad_sampling_params = SamplingParams(logprobs=2)
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
runner.generate(["Hello world"], sampling_params=bad_sampling_params)
+4 -3
View File
@@ -4,6 +4,7 @@
import pytest
from vllm import LLM, SamplingParams
from vllm.exceptions import VLLMValidationError
MODEL = "hmellor/tiny-random-LlamaForCausalLM"
PROMPT = "Hello my name is Robert and I"
@@ -161,15 +162,15 @@ def test_allowed_token_ids(llm):
assert output[0].outputs[0].token_ids[-1] == token_id
# Reject empty allowed_token_ids.
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
_ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[]))
# Reject negative token id.
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
_ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[-1]))
# Reject out of vocabulary.
with pytest.raises(ValueError):
with pytest.raises(VLLMValidationError):
_ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[10000000]))
@@ -5,6 +5,7 @@
import pytest
from vllm.config import StructuredOutputsConfig
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
pytestmark = pytest.mark.cpu_test
@@ -32,7 +33,7 @@ def test_structured_outputs_rejected_for_diffusion_models():
params = SamplingParams(
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
)
with pytest.raises(ValueError, match="not yet supported for diffusion"):
with pytest.raises(VLLMValidationError, match="not yet supported for diffusion"):
params._validate_structured_outputs(
_StubModelConfig(is_diffusion=True),
StructuredOutputsConfig(),
@@ -63,7 +64,7 @@ def test_degenerate_structured_outputs_rejected(structured_outputs, match):
rejected at request validation (-> 400) instead of reaching and crashing
the engine."""
params = SamplingParams(structured_outputs=structured_outputs)
with pytest.raises(ValueError, match=match):
with pytest.raises(VLLMValidationError, match=match):
params._validate_structured_outputs(
_StubModelConfig(is_diffusion=False),
StructuredOutputsConfig(),