[Bugfix][Frontend] Expose completion response schemas in OpenAPI (#60926)
Signed-off-by: Guan Luo <gluo@nvidia.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import schemathesis
|
||||
from fastapi.testclient import TestClient
|
||||
from hypothesis import HealthCheck, settings
|
||||
from jsonschema import Draft202012Validator
|
||||
from schemathesis import GenerationMode
|
||||
from schemathesis.checks import not_a_server_error
|
||||
from schemathesis.config import (
|
||||
@@ -30,6 +33,76 @@ DEFAULT_TIMEOUT_SECONDS: Final[int] = 10 * _ROCM_TIMEOUT_MULTIPLIER
|
||||
LONG_TIMEOUT_SECONDS: Final[int] = 60 * _ROCM_TIMEOUT_MULTIPLIER
|
||||
|
||||
|
||||
def test_openapi_exposes_completion_response_schemas() -> None:
|
||||
"""The published 200 contract of both completion APIs must reference the
|
||||
real response models, for the JSON body and for each SSE data chunk.
|
||||
|
||||
Uses the production app: FastAPI renames models used as both input and
|
||||
output (-Input/-Output), which a router-only app cannot show.
|
||||
"""
|
||||
from vllm.entrypoints.launchers.app import build_app
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
|
||||
args = Namespace(
|
||||
disable_fastapi_docs=False,
|
||||
enable_offline_docs=False,
|
||||
root_path="",
|
||||
allowed_origins=["*"],
|
||||
allow_credentials=False,
|
||||
allowed_methods=["*"],
|
||||
allowed_headers=["*"],
|
||||
api_key=None,
|
||||
enable_request_id_headers=False,
|
||||
enable_fault_tolerance=False,
|
||||
middleware=[],
|
||||
log_error_stack=False,
|
||||
)
|
||||
# No lifespan: schema generation does not need an inference engine.
|
||||
client = TestClient(build_app(args, supported_tasks=("generate",)))
|
||||
document = client.get("/openapi.json").json()
|
||||
message = {"role": "assistant", "content": "hi"}
|
||||
for path, unary, unary_choice, chunk, chunk_choice in (
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
ChatCompletionResponse,
|
||||
{"index": 0, "message": message},
|
||||
ChatCompletionStreamResponse,
|
||||
{"index": 0, "delta": {"content": "hi"}},
|
||||
),
|
||||
(
|
||||
"/v1/completions",
|
||||
CompletionResponse,
|
||||
{"index": 0, "text": "hi"},
|
||||
CompletionStreamResponse,
|
||||
{"index": 0, "text": "hi"},
|
||||
),
|
||||
):
|
||||
content = document["paths"][path]["post"]["responses"]["200"]["content"]
|
||||
data = content["text/event-stream"]["schema"]["properties"]["data"]
|
||||
assert data["anyOf"][0] == {"const": "[DONE]"}
|
||||
for schema, model, choice in (
|
||||
(content["application/json"]["schema"], unary, unary_choice),
|
||||
(data["anyOf"][1]["contentSchema"], chunk, chunk_choice),
|
||||
):
|
||||
assert schema == {"$ref": f"#/components/schemas/{model.__name__}"}
|
||||
validator = Draft202012Validator(
|
||||
{**schema, "components": document["components"]}
|
||||
)
|
||||
payload = model(model="m", choices=[choice], usage={}).model_dump(
|
||||
mode="json"
|
||||
)
|
||||
validator.validate(payload)
|
||||
payload["choices"] = "not an array"
|
||||
assert not validator.is_valid(payload)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, Request
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import StreamOptions
|
||||
from vllm.entrypoints.serve.utils import api_utils
|
||||
@@ -14,6 +16,55 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_with_cancellation_disconnect_skips_response_validation():
|
||||
"""A disconnect must cancel the handler without a response_model 500."""
|
||||
app = FastAPI()
|
||||
started = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
@app.post("/", response_model=str)
|
||||
@api_utils.with_cancellation
|
||||
async def handler(body: dict, raw_request: Request):
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Future()
|
||||
finally:
|
||||
cancelled.set()
|
||||
|
||||
messages = []
|
||||
body_sent = False
|
||||
|
||||
async def receive():
|
||||
nonlocal body_sent
|
||||
if not body_sent:
|
||||
body_sent = True
|
||||
return {"type": "http.request", "body": b"{}", "more_body": False}
|
||||
await started.wait()
|
||||
return {"type": "http.disconnect"}
|
||||
|
||||
async def send(message):
|
||||
messages.append(message)
|
||||
|
||||
scope = {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "POST",
|
||||
"scheme": "http",
|
||||
"path": "/",
|
||||
"raw_path": b"/",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"client": ("127.0.0.1", 1234),
|
||||
"server": ("test", 80),
|
||||
}
|
||||
await asyncio.wait_for(app(scope, receive, send), timeout=5)
|
||||
assert messages[0]["status"] == 200
|
||||
assert messages[1]["body"] == b""
|
||||
await asyncio.wait_for(cancelled.wait(), timeout=5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream_options", "expected"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user