[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack

- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 17:10:56 +07:00
co-authored by Hinata
parent bd70a61c20
commit 2066d3143f
1086 changed files with 122 additions and 291919 deletions
File diff suppressed because it is too large Load Diff
@@ -1,245 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import anthropic
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"2048",
"--enforce-eager",
"--enable-prompt-tokens-details",
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
"--served-model-name",
"claude-3-7-sonnet-latest",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client_anthropic() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_simple_messages(client: anthropic.AsyncAnthropic):
resp = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
messages=[{"role": "user", "content": "how are you!"}],
)
assert resp.stop_reason == "end_turn"
assert resp.role == "assistant"
print(f"Anthropic response: {resp.model_dump_json()}")
@pytest.mark.asyncio
async def test_system_message(client: anthropic.AsyncAnthropic):
resp = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
system="you are a helpful assistant",
messages=[{"role": "user", "content": "how are you!"}],
)
assert resp.stop_reason == "end_turn"
assert resp.role == "assistant"
print(f"Anthropic response: {resp.model_dump_json()}")
@pytest.mark.asyncio
async def test_anthropic_streaming(client: anthropic.AsyncAnthropic):
resp = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
messages=[{"role": "user", "content": "how are you!"}],
stream=True,
)
first_chunk = None
chunk_count = 0
async for chunk in resp:
chunk_count += 1
if first_chunk is None and chunk.type == "message_start":
first_chunk = chunk
print(chunk.model_dump_json())
assert chunk_count > 0
assert first_chunk is not None, "message_start chunk was never observed"
assert first_chunk.message is not None, "first chunk should include message"
assert first_chunk.message.usage is not None, (
"first chunk should include usage stats"
)
assert first_chunk.message.usage.output_tokens == 0
assert first_chunk.message.usage.input_tokens > 5
@pytest.mark.asyncio
async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic):
resp = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
messages=[
{"role": "user", "content": "What's the weather like in New York today?"}
],
tools=[
{
"name": "get_current_weather",
"description": "Useful for querying the weather in a specified city.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City or region, for example: "
"New York, London, Tokyo, etc.",
}
},
"required": ["location"],
},
}
],
stream=False,
)
assert resp.stop_reason == "tool_use"
assert resp.role == "assistant"
print(f"Anthropic response: {resp.model_dump_json()}")
@pytest.mark.asyncio
async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
resp = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
messages=[
{
"role": "user",
"content": "What's the weather like in New York today?",
}
],
tools=[
{
"name": "get_current_weather",
"description": "Useful for querying the weather in a specified city.",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City or region, for example: "
"New York, London, Tokyo, etc.",
}
},
"required": ["location"],
},
}
],
stream=True,
)
async for chunk in resp:
print(chunk.model_dump_json())
@pytest.mark.asyncio
async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic):
response = await client.messages.create(
model="claude-3-7-sonnet-latest",
max_tokens=1024,
messages=[
{
"role": "user",
"content": "Extract the key information from this email:"
"John Smith (john@example.com) is interested in our "
"Enterprise plan and wants to schedule a demo for next Tuesday at 2pm.",
}
],
output_config={
"format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
"plan_interest": {"type": "string"},
"demo_requested": {"type": "boolean"},
},
"required": ["name", "email", "plan_interest", "demo_requested"],
"additionalProperties": False,
},
}
},
)
print(response.content[0].text)
json_obj = json.loads(response.content[0].text)
for key in ["name", "email", "plan_interest", "demo_requested"]:
assert key in json_obj, f"Missing key in output: {key}"
@pytest.mark.asyncio
async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic):
async def get_stream_usage(resp):
prompt_tokens = None
usage = None
async for chunk in resp:
if (
chunk.type == "message_start"
and chunk.message is not None
and chunk.message.usage is not None
):
prompt_tokens = chunk.message.usage.input_tokens
elif chunk.type == "message_delta" and chunk.usage is not None:
usage = chunk.usage
assert usage is not None
assert usage.input_tokens >= 0
assert usage.output_tokens >= 0
cache_created = usage.cache_creation_input_tokens
cache_read = usage.cache_read_input_tokens
assert cache_read is not None
assert cache_created is not None
assert cache_created >= 0
assert cache_read >= 0
assert prompt_tokens == usage.input_tokens + cache_created + cache_read
return usage
request = dict(
model="claude-3-7-sonnet-latest",
max_tokens=1,
extra_body={"temperature": 0.0},
messages=[
{
"role": "user",
"content": "Cache coverage sentinel. " * 256
+ "Answer with exactly one word: ok.",
}
],
stream=True,
)
cold_usage = await get_stream_usage(await client.messages.create(**request))
assert cold_usage.cache_read_input_tokens == 0
assert cold_usage.cache_creation_input_tokens is not None
assert cold_usage.cache_creation_input_tokens > 0
warm_usage = await get_stream_usage(await client.messages.create(**request))
assert warm_usage.cache_read_input_tokens is not None
assert warm_usage.cache_read_input_tokens > 0
@@ -1,52 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Anthropic protocol exports used by serving.
Guards against Docker/nightly images shipping a stale protocol module that is
missing symbols imported by ``vllm.entrypoints.anthropic.serving`` (issue #44759).
"""
import pytest
from vllm.entrypoints.anthropic.protocol import (
AnthropicContentBlock,
AnthropicContextManagement,
AnthropicCountTokensRequest,
AnthropicCountTokensResponse,
AnthropicDelta,
AnthropicError,
AnthropicMessagesRequest,
AnthropicMessagesResponse,
AnthropicOutputConfig,
AnthropicStreamEvent,
AnthropicThinkingConfig,
AnthropicUsage,
)
pytestmark = pytest.mark.skip_global_cleanup
SERVING_PROTOCOL_EXPORTS = (
AnthropicContentBlock,
AnthropicContextManagement,
AnthropicCountTokensRequest,
AnthropicCountTokensResponse,
AnthropicDelta,
AnthropicError,
AnthropicMessagesRequest,
AnthropicMessagesResponse,
AnthropicOutputConfig,
AnthropicStreamEvent,
AnthropicThinkingConfig,
AnthropicUsage,
)
def test_serving_protocol_exports_are_importable():
for export in SERVING_PROTOCOL_EXPORTS:
assert export is not None
def test_anthropic_output_config_instantiation():
config = AnthropicOutputConfig()
assert config.effort is None
assert config.format is None
-693
View File
@@ -1,693 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``vllm/entrypoints/cohere/api_router.py``.
Covers:
* The optional-import guard: ``attach_router`` is a no-op when the
``cohere`` SDK isn't installed.
* The env-var opt-in gate: ``attach_router`` is a no-op unless
``VLLM_ENABLE_COHERE_API=1`` is set.
* The router wiring: response shapes (JSON + SSE), error translation,
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
Implemented).
* The render route (``POST /cohere/v2/chat/render``): that the v2 body
reaches :class:`ServingRender` as a converted ``ChatCompletionRequest``
and that the resulting ``GenerateRequest`` is returned verbatim.
"""
import json
from argparse import Namespace
from collections.abc import AsyncGenerator
from http import HTTPStatus
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.exceptions import RequestValidationError
from fastapi.testclient import TestClient
from vllm.entrypoints.cohere import api_router as api_router_mod
from vllm.entrypoints.cohere.api_router import attach_router
from vllm.entrypoints.cohere.protocol import (
AssistantMessageResponse,
CohereChatV2Response,
)
from vllm.entrypoints.cohere.serving import (
_API_SERVER_ONLY_TEMPLATE_KWARGS,
CohereServingChatV2,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
ReasoningParserKwargs,
)
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
from vllm.entrypoints.serve.exception_handling.handlers.http import (
http_exception_handler,
)
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
validation_exception_handler,
)
from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import (
vllm_error_handler,
)
from vllm.exceptions import VLLMError, VLLMValidationError
from vllm.sampling_params import SamplingParams
@pytest.fixture(autouse=True)
def _enable_cohere_api(monkeypatch):
"""Auto-enable the Cohere API gate for every test in this module.
The endpoint is opt-in in production (``VLLM_ENABLE_COHERE_API=1``);
every test in this file exercises the enabled path *except* the
dedicated gate test in :class:`TestEnvVarGate`, which unsets the
flag inside the test body.
"""
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
# ----------------------------------------------------------------------
# Fakes
# ----------------------------------------------------------------------
class _Handler:
"""Minimal stand-in for :class:`CohereServingChatV2` used by the
router. Each test sets ``self.result`` to either:
* a :class:`CohereChatV2Response` (non-streaming JSON path);
* an async generator yielding SSE frames (streaming path);
* an :class:`ErrorResponse` (error envelope path); or
* an exception (router-level error path).
"""
def __init__(self, result):
self.result = result
async def create_chat_v2(self, request, raw_request):
if isinstance(self.result, Exception):
raise self.result
return self.result
class _RenderChatHandler:
"""Stand-in for :class:`CohereServingChatV2` on the render route.
Only ``to_chat_completion_request`` is reachable from
``/cohere/v2/chat/render``, and it delegates to the real conversion
classmethod (which needs no engine or tokenizer) so the tests
exercise the actual v2 -> ``ChatCompletionRequest`` translation
rather than a stub of it.
"""
def to_chat_completion_request(self, request):
return CohereServingChatV2._convert_v2_to_chat_completion(request)
def _engine_chat_template_kwargs(self, chat_template_kwargs):
return CohereServingChatV2._engine_chat_template_kwargs(
self, # type: ignore[arg-type]
chat_template_kwargs,
)
class _RenderHandler:
"""Minimal stand-in for :class:`ServingRender`.
Records the ``ChatCompletionRequest`` it was handed so tests can
assert on the conversion, and returns ``self.result`` - either a
:class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception
to raise.
"""
def __init__(self, result):
self.result = result
self.seen_request = None
async def render_chat_request(self, request):
self.seen_request = request
if isinstance(self.result, Exception):
raise self.result
return self.result
def _generate_request() -> GenerateRequest:
return GenerateRequest(
request_id="chatcmpl-abc",
token_ids=[5, 6, 7],
sampling_params=SamplingParams(temperature=0.25),
model="m",
)
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
return app
def _build_render_app(
chat_handler, render_handler, *, enable_scale_out: bool = True
) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = chat_handler
app.state.serving_render = render_handler
return app
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
"""Build a FastAPI app that mirrors the real vLLM setup by installing
validation, HTTP, and vLLM exception handlers. The
:class:`CohereErrorEnvelopeMiddleware` registered by ``attach_router``
is expected to translate any resulting vLLM ``ErrorResponse`` body
into the ``CohereError`` wire shape.
"""
app = FastAPI()
# ``validation_exception_handler`` reads ``req.app.state.args``; the
# real cli builds this via argparse.
app.state.args = Namespace(
log_error_stack=False, enable_scale_out=True, tokens_only=False
)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(VLLMError)(vllm_error_handler)
return app
def _minimal_request_body() -> dict:
return {
"model": "m",
"messages": [{"role": "user", "content": "hi"}],
}
# ----------------------------------------------------------------------
# Optional-import guard
# ----------------------------------------------------------------------
class TestOptionalCohereImport:
"""``attach_router`` probes for the SDK once at module load (because
the route handler uses types imported from ``cohere``) and stashes
the result in ``_SDK_AVAILABLE``. Tests simulate the "SDK missing"
state by flipping that flag for the duration of the test.
``attach_router`` checks the env-var gate before the SDK probe, so
the SDK-missing branch is only reachable when the operator opts in
via ``VLLM_ENABLE_COHERE_API=1``. The flag-off-and-SDK-missing case
below exists to pin down that ordering — the flag-off short-circuits
"""
def test_flag_off_and_sdk_missing_stays_silent_about_sdk(self, monkeypatch, caplog):
"""Flag off doesn't do SDK-missing check.
When the operator hasn't opted in, ``attach_router`` must not
warn about the ``cohere`` SDK being missing: they never asked
for the endpoint, so surfacing the SDK gap is misleading noise.
Only the flag-off DEBUG message should fire.
"""
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
# The flag-off short-circuit ran; the SDK check never did.
assert not any(
"SDK is not installed" in rec.message for rec in caplog.records
), "SDK-missing log leaked despite the flag being off"
def test_flag_on_but_sdk_missing_logs_warning(self, monkeypatch, caplog):
"""Misconfiguration path: the operator explicitly opted into the
endpoint via ``VLLM_ENABLE_COHERE_API=1`` (already set by the
autouse fixture) but forgot to install ``cohere``.
"""
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
warn_sdk_records = [
rec
for rec in caplog.records
if "VLLM_ENABLE_COHERE_API=1" in rec.message
and "SDK is not installed" in rec.message
]
assert warn_sdk_records, (
"expected a WARNING that pairs the opt-in flag with the "
"missing SDK so operators notice the misconfiguration"
)
assert all(rec.levelname == "WARNING" for rec in warn_sdk_records)
def test_attach_router_registers_route_when_cohere_present(self):
app = _build_app(handler=None)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" in paths
# ----------------------------------------------------------------------
# VLLM_ENABLE_COHERE_API gate
# ----------------------------------------------------------------------
class TestEnvVarGate:
"""The Cohere v2 endpoint is opt-in via ``VLLM_ENABLE_COHERE_API``.
Even with the SDK installed, :func:`attach_router` must skip route
registration and middleware installation unless the env flag is
set. The autouse fixture on this module enables the flag by
default, so each test here explicitly disables it.
"""
def test_attach_router_noop_when_flag_unset(self, monkeypatch, caplog):
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
# The flag-off skip logs at DEBUG on purpose: this is the default
# state for every non-Cohere vLLM deployment, so an INFO log on
# every server startup would be pointless noise. The test raises
# caplog's level accordingly.
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
debug_flag_records = [
rec
for rec in caplog.records
if "VLLM_ENABLE_COHERE_API is not set" in rec.message
]
assert debug_flag_records, (
"expected a DEBUG message that the cohere flag is off"
)
assert all(rec.levelname == "DEBUG" for rec in debug_flag_records)
def test_attach_router_noop_when_flag_zero(self, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "0")
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
# ----------------------------------------------------------------------
# Endpoint behavior
# ----------------------------------------------------------------------
class TestEndpoint:
def test_501_when_handler_missing(self):
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
body = r.json()
assert "does not support" in body["message"]
assert "id" not in body # excluded by ``exclude_none=True``
def test_non_streaming_response_is_json(self):
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hello"}])
result = CohereChatV2Response(id="r1", finish_reason="COMPLETE", message=msg)
app = _build_app(handler=_Handler(result))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("application/json")
body = r.json()
assert body["id"] == "r1"
assert body["finish_reason"] == "COMPLETE"
assert body["message"]["content"][0]["text"] == "hello"
def test_streaming_response_is_sse(self):
async def _gen() -> AsyncGenerator[str, None]:
yield 'data: {"type":"message-start"}\n\n'
yield "data: [DONE]\n\n"
app = _build_app(handler=_Handler(_gen()))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={**_minimal_request_body(), "stream": True},
)
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("text/event-stream")
body = r.text
assert "message-start" in body
assert body.rstrip().endswith("[DONE]")
def test_error_response_translated_to_cohere_envelope(self):
err = ErrorResponse(
error=ErrorInfo(
message="bad request",
type="bad_request",
code=400,
)
)
app = _build_app(handler=_Handler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
assert body == {"message": "bad request"}
def test_handler_exception_returns_500_envelope(self):
app = _build_app(handler=_Handler(RuntimeError("kaboom")))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
body = r.json()
assert body == {"message": "kaboom"}
@pytest.mark.parametrize(
"exc",
[
VLLMValidationError("bad temperature", parameter="temperature"),
ValueError("bad temperature"),
],
ids=["vllm_validation_error", "value_error"],
)
def test_handler_client_error_returns_400_envelope(self, exc):
"""Client-caused handler errors are 4xx, matching the OpenAI routes."""
app = _build_app(handler=_Handler(exc))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
assert "bad temperature" in r.json()["message"]
def test_non_json_content_type_rejected(self):
"""The ``validate_json_request`` dependency raises
``RequestValidationError`` (HTTP 422) for non-JSON content
types, matching the behavior of the other vLLM API routers.
"""
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
content=json.dumps(_minimal_request_body()),
headers={"content-type": "text/plain"},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
def test_invalid_body_returns_422(self):
# ``model`` is required; omit it to trip Pydantic validation.
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={"messages": [{"role": "user", "content": "hi"}]},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
# ----------------------------------------------------------------------
# Render endpoint
# ----------------------------------------------------------------------
class TestRenderEndpoint:
"""``POST /cohere/v2/chat/render`` is the Cohere counterpart to
``POST /v1/chat/completions/render``: it converts the v2 body to a
``ChatCompletionRequest`` and hands it to the shared
``ServingRender``, returning the resulting ``GenerateRequest``
instead of running generation.
The route therefore spans *two* app-state handlers, and these tests
pin both halves: that the conversion preserves the Cohere-specific
request surface, and that the render handler's output (or error)
reaches the client unchanged.
"""
def test_route_registered(self):
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" in paths
def test_route_not_registered_when_flag_unset(self):
app = _build_app(handler=None, enable_scale_out=False)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" not in paths
assert "/cohere/v2/chat" in paths
def test_returns_generate_request_json(self):
render_handler = _RenderHandler(_generate_request())
app = _build_render_app(_RenderChatHandler(), render_handler)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("application/json")
body = r.json()
assert body["request_id"] == "chatcmpl-abc"
assert body["token_ids"] == [5, 6, 7]
assert body["sampling_params"]["temperature"] == 0.25
# The body must be a wire-valid GenerateRequest so callers can
# feed it straight back to /v1/generate.
assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7]
def test_cohere_request_surface_survives_conversion(self):
"""The point of routing through ``CohereServingChatV2`` rather
than asking callers to pre-convert: Cohere-only fields
(``documents``, ``safety_mode``, ``citation_options``) must land
in ``chat_template_kwargs`` so the rendered prompt matches what
``/cohere/v2/chat`` would have produced.
"""
render_handler = _RenderHandler(_generate_request())
app = _build_render_app(_RenderChatHandler(), render_handler)
body = {
**_minimal_request_body(),
"documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}],
"safety_mode": "STRICT",
"max_tokens": 32,
}
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=body)
assert r.status_code == HTTPStatus.OK
seen = render_handler.seen_request
assert seen is not None
assert seen.model == "m"
assert seen.messages[0]["content"] == "hi"
# ``max_tokens`` matters because ServingRender feeds it to
# ``get_max_tokens`` when building sampling params.
assert seen.max_completion_tokens == 32
kwargs = seen.chat_template_kwargs or {}
assert kwargs["documents"][0]["id"] == "doc-1"
# The conversion lower-cases safety_mode for the renderer.
assert kwargs["safety_mode"] == "strict"
def test_api_server_only_kwargs_not_forwarded(self):
"""As on ``/cohere/v2/chat``, the engine's reasoning parser must not
receive the template kwargs that only the API server reads."""
result = _generate_request()
result.reasoning_parser_kwargs = ReasoningParserKwargs(
chat_template_kwargs={
"safety_mode": "strict",
**dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"),
}
)
app = _build_render_app(_RenderChatHandler(), _RenderHandler(result))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.OK
assert r.json()["reasoning_parser_kwargs"] == {
"chat_template_kwargs": {"safety_mode": "strict"}
}
def test_501_when_chat_handler_missing(self):
app = _build_render_app(None, _RenderHandler(_generate_request()))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
assert "does not support" in r.json()["message"]
def test_501_when_render_handler_missing(self):
# A server without ``serving_render`` in app state (e.g. a
# non-generate task set) can't render, even though the Cohere
# chat handler exists.
app = _build_render_app(_RenderChatHandler(), None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
assert "does not support" in r.json()["message"]
def test_error_response_translated_to_cohere_envelope(self):
err = ErrorResponse(
error=ErrorInfo(message="unknown model", type="NotFound", code=404)
)
app = _build_render_app(_RenderChatHandler(), _RenderHandler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_FOUND
assert r.json() == {"message": "unknown model"}
def test_render_exception_returns_500_envelope(self):
app = _build_render_app(
_RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded"))
)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert r.json() == {"message": "tokenizer exploded"}
def test_conversion_client_error_returns_400_envelope(self):
"""Conversion runs inside the same ``try`` as the render call, so
a bad v2 body that slips past Pydantic still yields a 4xx in the
Cohere error envelope rather than an unhandled 500 with vLLM's shape.
"""
class _Boom:
def to_chat_completion_request(self, request):
raise ValueError("unconvertible message")
app = _build_render_app(_Boom(), _RenderHandler(_generate_request()))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
assert r.json() == {"message": "unconvertible message"}
def test_invalid_body_returns_422(self):
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat/render",
json={"messages": [{"role": "user", "content": "hi"}]},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
# ----------------------------------------------------------------------
# CohereErrorEnvelopeMiddleware
# ----------------------------------------------------------------------
class TestCohereErrorEnvelope:
"""When the app installs vLLM's global exception handlers, validation
and HTTP errors escape as ``ErrorResponse`` bodies before the route
handler runs. The middleware installed by ``attach_router`` must
normalise those bodies to the ``CohereError`` shape declared on the
endpoint's OpenAPI ``responses`` map so schema-conformance tests
(``test_openai_schema.py``) don't see a mismatch on ``/cohere/*``
responses.
"""
def test_validation_error_body_is_cohere_shaped(self):
# ``model=""`` and ``messages=[]`` trip our custom field
# validators, which raise VLLMValidationError and are routed
# through ``vllm_error_handler`` in the real vLLM
# server (producing the ``{"error": {...}}`` shape).
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json={"messages": [], "model": ""})
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# ``CohereError`` has ``message`` at the top level, not nested
# under an ``error`` envelope.
assert "error" not in body
assert "message" in body
assert isinstance(body["message"], str) and body["message"]
@pytest.mark.parametrize("path", ["/cohere/v2/chat", "/cohere/v2/chat/render"])
def test_required_tool_choice_without_tools_returns_400(self, path):
"""Reject an impossible tool choice before reaching either handler."""
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
response = client.post(
path,
json={**_minimal_request_body(), "tool_choice": "REQUIRED"},
)
assert response.status_code == HTTPStatus.BAD_REQUEST
body = response.json()
assert "error" not in body
assert "REQUIRED" in body["message"]
assert "tools" in body["message"]
def test_http_error_body_is_cohere_shaped(self):
# A raised ``HTTPException`` from anywhere in the request cycle
# is routed through ``http_exception_handler`` (producing the
# ``ErrorResponse`` shape) and must be translated.
app = _build_app_with_vllm_handlers(handler=None)
@app.get("/cohere/v2/boom")
async def _boom():
raise HTTPException(status_code=418, detail="teapot")
with TestClient(app) as client:
r = client.get("/cohere/v2/boom")
assert r.status_code == 418
body = r.json()
assert body == {"message": "teapot"}
def test_non_cohere_path_is_not_translated(self):
app = _build_app_with_vllm_handlers(handler=None)
@app.get("/v1/other")
async def _other():
raise HTTPException(status_code=400, detail="nope")
with TestClient(app) as client:
r = client.get("/v1/other")
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# Non-cohere paths keep the vLLM ``ErrorResponse`` shape.
assert "error" in body
assert body["error"]["message"] == "nope"
def test_streaming_response_passes_through(self):
# SSE responses have content-type text/event-stream; the
# middleware must never buffer these (which would break
# streaming) even though they're on ``/cohere/*``.
async def _gen() -> AsyncGenerator[str, None]:
yield 'data: {"type":"message-start"}\n\n'
yield "data: [DONE]\n\n"
app = _build_app_with_vllm_handlers(handler=_Handler(_gen()))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={**_minimal_request_body(), "stream": True},
)
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("text/event-stream")
assert "message-start" in r.text
assert r.text.rstrip().endswith("[DONE]")
def test_already_cohere_shaped_body_passes_through(self):
# When the handler returns an ``ErrorResponse`` the route
# itself translates it to ``CohereError``; the middleware sees
# the ``CohereError`` shape and must leave it alone.
err = ErrorResponse(
error=ErrorInfo(message="already cohere", type="Bad Request", code=400)
)
app = _build_app_with_vllm_handlers(handler=_Handler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# No ``error`` wrapper: the route already emitted the wire shape.
assert body == {"message": "already cohere"}
def test_request_id_preserved_in_translated_body(self):
# Client-provided ``X-Request-Id`` should be echoed as
# ``CohereError.id`` so callers can correlate failures.
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={"messages": [], "model": ""},
headers={"X-Request-Id": "req-abc"},
)
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
assert body.get("id") == "req-abc"
-343
View File
@@ -1,343 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end integration tests for the Cohere Chat v2 endpoints
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
These tests spin up a real ``vllm serve`` process via
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
contract over HTTP. They mirror the pattern used by
:mod:`tests.entrypoints.anthropic.test_messages`.
Two layers of integration are covered:
1. Raw HTTP via :mod:`httpx` — always runs, verifies the wire contract.
2. Cohere SDK (``pip install cohere``) — auto-skipped when the optional
dependency isn't installed, verifies SDK-level interop.
We use a tiny generic chat model (``HuggingFaceTB/SmolLM2-135M-Instruct``,
≈135M params) so the fixture stays runnable on CPU-only laptops. The
Cohere v2 endpoint is model-agnostic and just performs the v2 ↔ OpenAI
chat-completion translation, so the choice of model only matters for
response *shape* — the translation logic itself is unit-tested
elsewhere.
"""
import json
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# Tiny chat-tuned model so the fixture is cheap to boot on CPU-only
# hosts (Mac, CI runners without GPUs). The Cohere v2 translation is
# completely independent of which underlying chat model is loaded.
MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct"
SERVED_MODEL_NAME = "command-r-plus-08-2024"
@pytest.fixture(scope="module")
def server():
args = [
# 1k tokens is plenty for the prompts these tests send; trimming
# it from the default keeps the KV-cache footprint tiny.
"--max-model-len",
"1024",
"--max-num-seqs",
"4",
"--dtype",
"bfloat16",
"--enforce-eager",
# Advertise a Cohere model name so Cohere SDK ``model=`` calls
# round-trip without the ``model not found`` check.
"--served-model-name",
SERVED_MODEL_NAME,
# Disable the reasoning-model path so the test doesn't require
# the conversation to surface a thinking block (SmolLM2 doesn't
# emit Cohere-style reasoning tokens out of the box).
"--no-cohere-is-reasoning-model",
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
"--enable-scale-out",
]
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
# startup. The default (4 GB) trips Mac's RAM check on smaller
# machines; 1 GB is more than enough for max_model_len=1024 and
# max_num_seqs=4.
# ``VLLM_ENABLE_COHERE_API=1`` flips the opt-in gate that
# ``vllm.entrypoints.cohere.api_router.attach_router`` reads to
# decide whether to register ``POST /cohere/v2/chat`` at server
# startup. Without it the fixture would boot a server that returns
# 404 for every request the tests below make.
env_dict = {
"VLLM_CPU_KVCACHE_SPACE": "1",
"VLLM_ENABLE_COHERE_API": "1",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
yield remote_server
# ----------------------------------------------------------------------
# Layer 1: raw HTTP contract (no optional deps)
# ----------------------------------------------------------------------
@pytest_asyncio.fixture
async def httpx_client(server):
async with httpx.AsyncClient(
base_url=server.url_root, timeout=httpx.Timeout(120.0)
) as client:
yield client
@pytest.mark.asyncio
async def test_cohere_v2_chat_non_streaming(httpx_client: httpx.AsyncClient):
resp = await httpx_client.post(
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Say hi."}],
"max_tokens": 16,
"stream": False,
},
)
assert resp.status_code == 200, resp.text
payload = resp.json()
# The response envelope follows Cohere v2's schema.
assert "message" in payload
assert payload["message"]["role"] == "assistant"
# ``content`` is a list of content blocks; the synthesized
# ``CohereServingChatV2`` should always emit at least one ``text``
# block (it falls back to an empty block when the model returned
# nothing).
content = payload["message"]["content"]
assert isinstance(content, list) and len(content) >= 1
assert content[0]["type"] == "text"
# ``usage`` is always populated by the translator.
assert "usage" in payload
assert "billed_units" in payload["usage"]
assert "tokens" in payload["usage"]
# ``finish_reason`` is one of Cohere's enum values.
assert payload["finish_reason"] in {
"COMPLETE",
"MAX_TOKENS",
"STOP_SEQUENCE",
"TOOL_CALL",
"ERROR",
}
@pytest.mark.asyncio
async def test_cohere_v2_chat_streaming(httpx_client: httpx.AsyncClient):
"""Streaming returns SSE frames in the v2 message-lifecycle shape."""
events: list[dict] = []
async with httpx_client.stream(
"POST",
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Hi"}],
"max_tokens": 8,
"stream": True,
},
) as resp:
assert resp.status_code == 200, await resp.aread()
assert resp.headers["content-type"].startswith("text/event-stream")
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
data = line[len("data: ") :]
if data == "[DONE]":
events.append({"type": "_DONE_"})
continue
events.append(json.loads(data))
types = [ev["type"] for ev in events]
# The lifecycle always starts with message-start and ends with [DONE]
# preceded by message-end.
assert types[0] == "message-start"
assert types[-1] == "_DONE_"
assert types[-2] == "message-end"
# message-start must carry the chunk/message id.
assert events[0].get("id")
@pytest.mark.asyncio
async def test_cohere_v2_chat_validation_error_returns_400(
httpx_client: httpx.AsyncClient,
):
# Missing required ``model`` field → FastAPI/Pydantic returns 400.
resp = await httpx_client.post(
"/cohere/v2/chat",
json={"messages": []},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
"""``/cohere/v2/chat/render`` tokenizes without running the model.
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
contract that makes the endpoint useful: v2 sampling fields
(``temperature``, ``stop_sequences``) survive the v2 ->
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
render once and generate later without re-deriving them.
"""
resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Say hi."}],
"max_tokens": 16,
"temperature": 0.125,
"stop_sequences": ["END"],
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
token_ids = data["token_ids"]
assert isinstance(token_ids, list) and len(token_ids) > 0
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
assert data["request_id"]
sampling_params = data["sampling_params"]
assert sampling_params["temperature"] == 0.125
assert sampling_params["stop"] == ["END"]
# No generation happened, so the response carries none of the chat
# envelope the /cohere/v2/chat endpoint would return.
assert "message" not in data
assert "finish_reason" not in data
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
httpx_client: httpx.AsyncClient,
):
"""The rendered ids are the real templated prompt, not a debug dump.
Round-tripping them through ``/detokenize`` and finding the user
message back proves the endpoint applied the chat template and
tokenized it, which is what a caller splitting prompt construction
from generation depends on.
"""
render_resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"max_tokens": 8,
},
)
assert render_resp.status_code == 200, render_resp.text
token_ids = render_resp.json()["token_ids"]
detok_resp = await httpx_client.post(
"/detokenize",
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
)
assert detok_resp.status_code == 200, detok_resp.text
prompt = detok_resp.json()["prompt"]
assert "What is the capital of France?" in prompt
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_validation_error_returns_400(
httpx_client: httpx.AsyncClient,
):
"""Bad bodies get the Cohere error envelope, not vLLM's."""
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
assert resp.status_code == 400
body = resp.json()
assert "error" not in body
assert body.get("message")
@pytest.mark.asyncio
async def test_cohere_v2_chat_documents_field_accepted(
httpx_client: httpx.AsyncClient,
):
"""The v2 endpoint forwards ``documents`` into chat_template_kwargs.
We only assert the request is accepted and produces a 200 response —
the renderer-level effect is covered by ``tests/renderers/test_cohere.py``.
"""
resp = await httpx_client.post(
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Summarize."}],
"documents": [{"id": "d1", "data": {"title": "T", "snippet": "S"}}],
"max_tokens": 16,
"stream": False,
},
)
assert resp.status_code == 200, resp.text
# ----------------------------------------------------------------------
# Layer 2: Cohere SDK round-trip (auto-skipped if SDK absent)
# ----------------------------------------------------------------------
@pytest_asyncio.fixture
async def cohere_async_client(server):
cohere = pytest.importorskip("cohere")
# The vLLM endpoint is mounted at ``/cohere/v2/chat`` while the
# cohere SDK targets ``${base_url}/v2/chat``; point base_url at the
# ``/cohere`` prefix so paths line up.
client = cohere.AsyncClientV2(
api_key="dummy",
base_url=server.url_for("cohere"),
)
try:
yield client
finally:
# ``AsyncClientV2`` exposes a sync close; if a future version
# adds aclose we still close cleanly.
close = getattr(client, "aclose", None) or getattr(client, "close", None)
if close is not None:
result = close()
if hasattr(result, "__await__"):
await result
@pytest.mark.asyncio
async def test_cohere_sdk_non_streaming(cohere_async_client):
resp = await cohere_async_client.chat(
model=SERVED_MODEL_NAME,
messages=[{"role": "user", "content": "Say hi."}],
max_tokens=16,
)
# SDK parses our JSON into typed objects.
assert resp.message.role == "assistant"
assert resp.message.content is not None
assert len(resp.message.content) >= 1
assert resp.message.content[0].type == "text"
assert resp.finish_reason in {
"COMPLETE",
"MAX_TOKENS",
"STOP_SEQUENCE",
"TOOL_CALL",
"ERROR",
}
@pytest.mark.asyncio
async def test_cohere_sdk_streaming(cohere_async_client):
events: list[str] = []
stream = cohere_async_client.chat_stream(
model=SERVED_MODEL_NAME,
messages=[{"role": "user", "content": "Hi"}],
max_tokens=8,
)
async for ev in stream:
events.append(ev.type)
assert events, "SDK stream yielded no events"
assert events[0] == "message-start"
assert events[-1] == "message-end"
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for :mod:`vllm.entrypoints.cohere.cohere_chat_message`.
The module hosts the citation-carrying subclasses of the OpenAI chat
completion protocol (``CohereChatMessage`` / ``CohereDeltaMessage``) plus
the shared ``Citation`` / ``CitationSource`` shapes. These tests pin:
1. Serialization behavior we own (empty-``citations`` cleanup on the
subclass ``_serialize``, ``exclude_none`` shape, round-trip).
2. The constraint that ``type`` discriminators reject unknown values.
"""
import pytest
from pydantic import ValidationError
from vllm.entrypoints.cohere.cohere_chat_message import (
Citation,
CitationSource,
CohereChatMessage,
CohereDeltaMessage,
)
# ======================================================================
# CitationSource
# ======================================================================
class TestCitationSource:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
CitationSource(type="other")
def test_none_fields_excluded_from_dump(self):
s = CitationSource(type="document", id="d1")
assert s.model_dump(exclude_none=True) == {
"type": "document",
"id": "d1",
}
def test_resolved_source_survives_json_round_trip(self):
# The parser produces fully-resolved sources (see
# ``_melody_sources_to_vllm`` in
# ``vllm/reasoning/cohere_command_reasoning_parser.py``) which
# then flow through the internal streaming pipeline (parser ->
# OpenAI stream chunk -> ``_chat_completion_stream_to_v2``).
# Every field must round-trip through ``model_dump_json`` at
# that layer or the wire event ends up missing pieces.
src = CitationSource(
type="tool",
id="res_a0",
tool_output={"id": "res_a0", "text": "r"},
)
src2 = CitationSource.model_validate_json(src.model_dump_json())
assert src2.type == "tool"
assert src2.id == "res_a0"
assert src2.tool_output == {"id": "res_a0", "text": "r"}
# ======================================================================
# Citation
# ======================================================================
class TestCitation:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
Citation(type="OTHER")
def test_dump_excludes_none_fields(self):
c = Citation(start=0, end=5, text="hello")
dumped = c.model_dump(exclude_none=True)
assert dumped == {
"start": 0,
"end": 5,
"text": "hello",
"sources": [],
}
# ======================================================================
# CohereDeltaMessage
# ======================================================================
class TestCohereDeltaMessage:
def test_citations_omitted_from_dump_when_none(self):
# Non-grounded deltas should never carry ``citations: null`` on
# the wire even when routed through the Cohere subclass.
d = CohereDeltaMessage(content="hello")
dumped = d.model_dump(exclude_none=True)
assert "citations" not in dumped
def test_citations_dump_round_trip(self):
d = CohereDeltaMessage(citations=[Citation(start=0, end=5, text="hi")])
dumped = d.model_dump(exclude_none=True)
assert dumped["citations"][0]["text"] == "hi"
# Round-trip through the subclass to confirm the field validates.
d2 = CohereDeltaMessage.model_validate(dumped)
assert d2.citations[0].text == "hi"
# ======================================================================
# CohereChatMessage
# ======================================================================
class TestCohereChatMessage:
def test_citations_omitted_from_dump_when_none(self):
m = CohereChatMessage(role="assistant", content="hi")
dumped = m.model_dump(exclude_none=True)
assert "citations" not in dumped
def test_citations_dump_round_trip(self):
m = CohereChatMessage(
role="assistant",
content="hello",
citations=[
Citation(
start=0,
end=5,
text="hello",
sources=[CitationSource(type="document", id="d1")],
)
],
)
dumped = m.model_dump(exclude_none=True)
assert dumped["role"] == "assistant"
assert dumped["content"] == "hello"
assert dumped["citations"][0]["text"] == "hello"
assert dumped["citations"][0]["sources"][0]["id"] == "d1"
# Round-trip through the subclass to confirm the field validates.
m2 = CohereChatMessage.model_validate(dumped)
assert m2.citations[0].text == "hello"
assert m2.citations[0].sources[0].id == "d1"
-271
View File
@@ -1,271 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``vllm/entrypoints/cohere/protocol.py``.
The module is mostly a thin wrapper around the official ``cohere`` SDK
types plus a few local additions:
* :class:`CohereError` envelope.
* :class:`CohereChatV2Request` (model and messages required,
``max_tokens`` non-negative).
* :class:`CohereChatV2Response` plus the usage / logprob helpers.
* The streaming event subclasses that bake a wire-format ``type``
discriminator into ``model_dump()`` so SSE consumers can demux on it.
"""
import pytest
from pydantic import ValidationError
import vllm.envs as envs
from vllm.entrypoints.cohere.protocol import (
AssistantMessageResponse,
CitationEndEvent,
CitationStartEvent,
CohereChatV2Request,
CohereChatV2Response,
CohereError,
CohereUsageTokens,
ContentDeltaEvent,
ContentEndEvent,
ContentStartEvent,
MessageEndEvent,
MessageStartEvent,
ToolCallDeltaEvent,
ToolCallEndEvent,
ToolCallStartEvent,
ToolPlanDeltaEvent,
)
from vllm.exceptions import VLLMValidationError
# ======================================================================
# CohereError
# ======================================================================
class TestCohereError:
def test_model_dump_excludes_none(self):
err = CohereError(message="boom")
assert err.model_dump(exclude_none=True) == {"message": "boom"}
# ======================================================================
# CohereChatV2Request
# ======================================================================
class TestCohereChatV2Request:
def test_empty_model_rejected(self):
with pytest.raises(VLLMValidationError, match="model is required") as exc_info:
CohereChatV2Request(model="", messages=[{"role": "user", "content": "hi"}])
assert exc_info.value.parameter == "model"
assert exc_info.value.value is None
def test_negative_max_tokens_rejected(self):
with pytest.raises(VLLMValidationError, match="non-negative") as exc_info:
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
max_tokens=-1,
)
assert exc_info.value.parameter == "max_tokens"
assert exc_info.value.value == -1
def test_empty_messages_rejected(self):
with pytest.raises(
VLLMValidationError, match="messages must contain at least one message"
) as exc_info:
CohereChatV2Request(model="m", messages=[])
assert exc_info.value.parameter == "messages"
assert exc_info.value.value is None
def test_zero_max_tokens_allowed(self):
# Zero is allowed (the docs allow 0 -> return prompt only).
req = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
max_tokens=0,
)
assert req.max_tokens == 0
def test_invalid_tool_choice_rejected(self):
with pytest.raises(ValidationError):
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice="ANY", # not REQUIRED/NONE
)
@pytest.mark.parametrize("tool_fields", [{}, {"tools": None}, {"tools": []}])
def test_required_tool_choice_requires_tools(self, tool_fields):
with pytest.raises(VLLMValidationError, match="REQUIRED.*tools") as exc_info:
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice="REQUIRED",
**tool_fields,
)
assert exc_info.value.parameter == "tool_choice"
@pytest.mark.parametrize("tool_choice", [None, "NONE"])
def test_nonrequired_tool_choice_allows_empty_tools(self, tool_choice):
request = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice=tool_choice,
tools=[],
)
assert request.tool_choice == tool_choice
def test_stop_sequences_bounded_by_env_limit(self):
messages = [{"role": "user", "content": "hi"}]
limit = envs.VLLM_MAX_STOP_STRINGS
req = CohereChatV2Request(
model="m",
messages=messages,
stop_sequences=[f"s{i}" for i in range(limit)],
)
assert len(req.stop_sequences) == limit
with pytest.raises(ValidationError):
CohereChatV2Request(
model="m",
messages=messages,
stop_sequences=[f"s{i}" for i in range(limit + 1)],
)
# ======================================================================
# Usage / Logprobs
# ======================================================================
class TestUsage:
def test_tokens_serialization(self):
u = CohereUsageTokens(input_tokens=10, output_tokens=5)
assert u.model_dump() == {"input_tokens": 10.0, "output_tokens": 5.0}
# ======================================================================
# CohereChatV2Response
# ======================================================================
class TestCohereChatV2Response:
def test_invalid_finish_reason_rejected(self):
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hi"}])
with pytest.raises(ValidationError):
CohereChatV2Response(
id="r1",
finish_reason="NOT_A_REASON",
message=msg,
)
# ======================================================================
# Streaming event ``type`` discriminator baked into model_dump()
# ======================================================================
class TestStreamingEventTypeField:
"""Each event subclass adds a ``type: Literal[...]`` field with a
default so ``model_dump()`` always emits the wire-format discriminator
(the parent SDK classes don't declare ``type`` as a Pydantic field).
"""
@pytest.mark.parametrize(
"cls, expected_type, kwargs",
[
(
MessageStartEvent,
"message-start",
{"id": "a", "delta": {"message": {"role": "assistant"}}},
),
(
ContentStartEvent,
"content-start",
{
"index": 0,
"delta": {"message": {"content": {"type": "text", "text": ""}}},
},
),
(
ContentDeltaEvent,
"content-delta",
{
"index": 0,
"delta": {"message": {"content": {"text": "hi"}}},
},
),
(ContentEndEvent, "content-end", {"index": 0}),
(
ToolPlanDeltaEvent,
"tool-plan-delta",
{"delta": {"message": {"tool_plan": "thinking"}}},
),
(
ToolCallStartEvent,
"tool-call-start",
{
"index": 0,
"delta": {
"message": {
"tool_calls": {
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": ""},
}
}
},
},
),
(
ToolCallDeltaEvent,
"tool-call-delta",
{
"index": 0,
"delta": {
"message": {"tool_calls": {"function": {"arguments": "{}"}}}
},
},
),
(ToolCallEndEvent, "tool-call-end", {"index": 0}),
(
CitationStartEvent,
"citation-start",
{
"index": 0,
"delta": {
"message": {
"citations": {"start": 0, "end": 5, "text": "hello"}
}
},
},
),
(CitationEndEvent, "citation-end", {"index": 0}),
(
MessageEndEvent,
"message-end",
{"id": "a", "delta": {"finish_reason": "COMPLETE"}},
),
],
)
def test_type_field_default(self, cls, expected_type, kwargs):
ev = cls(**kwargs)
# type field is auto-populated from the Literal default.
assert ev.type == expected_type
# The discriminator must be present in the serialized payload so
# clients reading the stream can demux on it.
dumped = ev.model_dump(exclude_none=True)
assert dumped["type"] == expected_type
# Same in JSON form (what ``_emit`` serializes).
assert f'"type":"{expected_type}"' in ev.model_dump_json(exclude_none=True)
def test_type_field_cannot_be_overridden_to_wrong_value(self):
# Literal types reject any value other than the bake-in default.
with pytest.raises(ValidationError):
MessageStartEvent(
id="a",
delta={"message": {"role": "assistant"}},
type="other",
)
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sanity tests for the small registry / CLI-arg / config additions made
by the Cohere v2 chat API:
* ``vllm/renderers/registry.py``: new ``"cohere"`` renderer entry.
* ``vllm/tokenizers/registry.py``: new ``"cohere"`` tokenizer entry
(aliased to the cached HF tokenizer).
* ``vllm/entrypoints/openai/cli_args.py``: new
``cohere_is_reasoning_model`` field.
* ``vllm/config/model.py``: ``"cohere"`` added to ``TokenizerMode``
Literal.
"""
import dataclasses
import typing
import pytest
from vllm.config.model import TokenizerMode
from vllm.entrypoints.launchers.cli_args import BaseFrontendArgs, make_arg_parser
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.registry import TokenizerRegistry
from vllm.utils.argparse_utils import FlexibleArgumentParser
class TestRendererRegistry:
def test_cohere_renderer_registered(self):
# The registry resolves to the importable ``CohereRenderer`` class.
cls = RENDERER_REGISTRY.load_renderer_cls("cohere")
assert cls.__name__ == "CohereRenderer"
# Sanity: class lives in the cohere renderer module.
assert cls.__module__ == "vllm.renderers.cohere"
class TestTokenizerRegistry:
def test_cohere_aliased_to_cached_hf_tokenizer(self):
# ``cohere`` mode uses the standard HF tokenizer; only the
# renderer stage is replaced. This test guards against accidental
# divergence.
cls = TokenizerRegistry.load_tokenizer_cls("cohere")
assert cls.__name__ == "CachedHfTokenizer"
class TestTokenizerModeLiteral:
def test_cohere_is_a_valid_tokenizer_mode(self):
# The Literal must enumerate ``"cohere"`` so engine arg parsing
# accepts ``--tokenizer-mode cohere``.
modes = typing.get_args(TokenizerMode)
assert "cohere" in modes
# ----------------------------------------------------------------------
# ``--cohere-is-reasoning-model`` CLI flag
# ----------------------------------------------------------------------
class TestCohereCliArg:
"""Verifies the new ``--cohere-is-reasoning-model`` flag end-to-end
through ``make_arg_parser`` — mirrors the pattern used in
:mod:`tests.entrypoints.openai.test_cli_args`.
"""
def test_default_value_on_dataclass_is_true(self):
fields = {f.name: f for f in dataclasses.fields(BaseFrontendArgs)}
assert "cohere_is_reasoning_model" in fields
field = fields["cohere_is_reasoning_model"]
assert field.default is True
assert field.type is bool
@pytest.fixture
def serve_parser(self) -> FlexibleArgumentParser:
parser = FlexibleArgumentParser()
return make_arg_parser(parser)
def test_default_via_argparse_is_true(self, serve_parser: FlexibleArgumentParser):
# No flag supplied → dataclass default (True) wins.
args = serve_parser.parse_args(["--model", "m"])
assert args.cohere_is_reasoning_model is True
def test_explicit_false_via_argparse(self, serve_parser: FlexibleArgumentParser):
# Boolean dataclass fields are wired up as ``--flag value`` /
# ``--no-flag`` pairs by FlexibleArgumentParser.
args = serve_parser.parse_args(
["--model", "m", "--no-cohere-is-reasoning-model"]
)
assert args.cohere_is_reasoning_model is False
def test_explicit_true_via_argparse(self, serve_parser: FlexibleArgumentParser):
args = serve_parser.parse_args(["--model", "m", "--cohere-is-reasoning-model"])
assert args.cohere_is_reasoning_model is True
File diff suppressed because it is too large Load Diff
@@ -1,952 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Cohere v2 SSE stream conversion in
``vllm/entrypoints/cohere/serving.py``.
The stream-translation entry point is
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
async iterable of OpenAI chunk objects into Cohere's
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
event stream.
We test the helpers (``_StreamState``, ``_handle_*_delta``, etc.) in
isolation, plus a handful of end-to-end scenarios that exercise the
state machine.
"""
import json
from collections.abc import AsyncGenerator, Sequence
from typing import Any
import pytest
from vllm.entrypoints.cohere.cohere_chat_message import (
Citation as VLLMCitation,
)
from vllm.entrypoints.cohere.cohere_chat_message import (
CitationSource,
)
from vllm.entrypoints.cohere.protocol import MessageStartEvent
from vllm.entrypoints.cohere.serving import (
_DONE_FRAME,
CohereServingChatV2,
ContentBlockType,
_emit,
_sse,
_StreamState,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
class _FakeServing(CohereServingChatV2):
"""Lightweight stand-in for :class:`CohereServingChatV2` that skips
the heavy ``OpenAIServingChat.__init__`` chain (which would need a
real engine client, model registry, render service, etc.).
Only ``_is_reasoning_model`` is read by the methods under test
(``_chat_completion_stream_to_v2`` and the per-delta handlers); the
rest is dead weight for unit testing.
"""
def __init__(self, is_reasoning_model: bool = True) -> None:
# Intentionally skipping super().__init__ — see class docstring.
self._is_reasoning_model = is_reasoning_model
def _serving(is_reasoning_model: bool = True) -> CohereServingChatV2:
return _FakeServing(is_reasoning_model=is_reasoning_model)
def _parse_event(frame: str) -> dict[str, Any]:
"""Strip the ``data: ... \\n\\n`` wrapper and parse the JSON payload."""
assert frame.startswith("data: ")
assert frame.endswith("\n\n")
return json.loads(frame[len("data: ") : -2])
def _make_chunk(
*,
chunk_id: str = "chunk_0",
role: str | None = None,
content: str | None = None,
reasoning: str | None = None,
tool_calls: list[dict[str, Any]] | None = None,
finish_reason: str | None = None,
stop_reason: int | str | None = None,
usage: dict[str, Any] | None = None,
omit_choices: bool = False,
citations: list[Any] | None = None,
) -> ChatCompletionStreamResponse:
"""Build the raw chat chunk the production converter consumes."""
delta: dict[str, Any] = {}
if role is not None:
delta["role"] = role
if content is not None:
delta["content"] = content
if reasoning is not None:
delta["reasoning"] = reasoning
if tool_calls is not None:
delta["tool_calls"] = tool_calls
if citations is not None:
delta["citations"] = citations
payload: dict[str, Any] = {
"id": chunk_id,
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
}
if not omit_choices:
payload["choices"] = [
{
"index": 0,
"delta": delta,
"finish_reason": finish_reason,
"stop_reason": stop_reason,
}
]
else:
payload["choices"] = []
if usage is not None:
payload["usage"] = usage
return ChatCompletionStreamResponse.model_validate(payload)
async def _stream_from(
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
for item in items:
yield item
async def _drain(
serving: CohereServingChatV2,
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> list[str]:
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
the emitted SSE frames."""
gen = serving._chat_completion_stream_to_v2(_stream_from(items))
return [frame async for frame in gen]
# ======================================================================
# Low-level helpers: _sse, _emit, _DONE_FRAME
# ======================================================================
class TestSSEHelpers:
def test_sse_wraps_payload(self):
assert _sse("hello") == "data: hello\n\n"
def test_done_frame_constant(self):
# cohere-python and Fern-generated clients key off this exact
# sentinel; keep it byte-for-byte stable.
assert _DONE_FRAME == "data: [DONE]\n\n"
def test_emit_serializes_event_with_type_discriminator(self):
frame = _emit(
MessageStartEvent(id="abc", delta={"message": {"role": "assistant"}})
)
payload = _parse_event(frame)
assert payload["type"] == "message-start"
assert payload["id"] == "abc"
assert payload["delta"] == {"message": {"role": "assistant"}}
# ======================================================================
# _StreamState
# ======================================================================
class TestStreamState:
def test_content_index_monotonic(self):
st = _StreamState()
assert st.next_content_index() == 0
assert st.next_content_index() == 1
assert st.next_content_index() == 2
def test_citation_index_separate_from_content_index(self):
st = _StreamState()
st.next_content_index() # 0
st.next_content_index() # 1
# Citation indexing is independent of content indexing.
assert st.next_citation_index() == 0
assert st.next_citation_index() == 1
# ======================================================================
# _close_open_blocks
# ======================================================================
class TestCloseOpenBlocks:
def test_no_block_open_emits_nothing(self):
serving = _serving()
state = _StreamState()
assert serving._close_open_blocks(state) == []
def test_text_block_emits_content_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.TEXT
state.active_block_index = 1
out = serving._close_open_blocks(state)
assert len(out) == 1
payload = _parse_event(out[0])
assert payload == {"type": "content-end", "index": 1}
assert state.active_block is None
assert state.active_block_index is None
def test_thinking_block_emits_content_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.THINKING
state.active_block_index = 3
out = serving._close_open_blocks(state)
payload = _parse_event(out[0])
assert payload == {"type": "content-end", "index": 3}
def test_tool_call_block_emits_tool_call_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.TOOL_CALL
state.active_tool_index = 7
out = serving._close_open_blocks(state)
payload = _parse_event(out[0])
assert payload == {"type": "tool-call-end", "index": 7}
assert state.active_tool_index is None
# ======================================================================
# _handle_text_delta
# ======================================================================
class TestHandleTextDelta:
def test_opens_block_first_time(self):
serving = _serving()
state = _StreamState()
events = serving._handle_text_delta(state, "Hi")
assert len(events) == 2
start = _parse_event(events[0])
delta = _parse_event(events[1])
assert start["type"] == "content-start"
assert start["index"] == 0
assert start["delta"]["message"]["content"]["type"] == "text"
assert delta["type"] == "content-delta"
assert delta["index"] == 0
assert delta["delta"]["message"]["content"]["text"] == "Hi"
assert state.active_block == ContentBlockType.TEXT
assert state.active_block_index == 0
def test_continues_block_with_just_delta(self):
serving = _serving()
state = _StreamState()
serving._handle_text_delta(state, "Hi")
events = serving._handle_text_delta(state, " there")
# Only a delta event, no new content-start.
assert len(events) == 1
delta = _parse_event(events[0])
assert delta["type"] == "content-delta"
assert delta["delta"]["message"]["content"]["text"] == " there"
def test_switches_from_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
# Open a thinking block first, then switch to text.
serving._handle_thinking_delta(state, "ponder")
events = serving._handle_text_delta(state, "answer")
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "content-start", "content-delta"]
# The text block gets a new index (1), distinct from thinking's 0.
assert _parse_event(events[1])["index"] == 1
assert state.active_block == ContentBlockType.TEXT
# ======================================================================
# _handle_thinking_delta
# ======================================================================
class TestHandleThinkingDelta:
def test_reasoning_model_opens_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
events = serving._handle_thinking_delta(state, "thought")
assert len(events) == 2
start = _parse_event(events[0])
delta = _parse_event(events[1])
assert start["type"] == "content-start"
assert start["delta"]["message"]["content"]["type"] == "thinking"
assert delta["type"] == "content-delta"
assert delta["delta"]["message"]["content"]["thinking"] == "thought"
assert state.active_block == ContentBlockType.THINKING
def test_reasoning_model_continues_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
serving._handle_thinking_delta(state, "first")
events = serving._handle_thinking_delta(state, " more")
assert len(events) == 1
assert _parse_event(events[0])["type"] == "content-delta"
def test_non_reasoning_model_emits_tool_plan_delta(self):
# Older Command models stream reasoning as ``tool_plan`` deltas;
# no content-start/end pair is emitted.
serving = _serving(is_reasoning_model=False)
state = _StreamState()
events = serving._handle_thinking_delta(state, "planning")
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-plan-delta"
assert payload["delta"]["message"]["tool_plan"] == "planning"
# ``tool_plan`` deltas don't claim an active content block.
assert state.active_block is None
def test_non_reasoning_model_closes_open_text_block(self):
serving = _serving(is_reasoning_model=False)
state = _StreamState()
# Open a text block first.
serving._handle_text_delta(state, "answer")
events = serving._handle_thinking_delta(state, "rethink")
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "tool-plan-delta"]
# ======================================================================
# _handle_tool_call_deltas
# ======================================================================
class TestHandleToolCallDeltas:
def test_new_tool_call_opens_tool_call_start(self):
serving = _serving()
state = _StreamState()
deltas = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type(
"Fn", (), {"name": "calc", "arguments": '{"x":'}
)(),
},
)()
]
events = serving._handle_tool_call_deltas(state, deltas)
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-call-start"
assert payload["index"] == 0
tc = payload["delta"]["message"]["tool_calls"]
assert tc["id"] == "c1"
assert tc["function"]["name"] == "calc"
assert tc["function"]["arguments"] == '{"x":'
assert state.active_block == ContentBlockType.TOOL_CALL
assert state.active_tool_index == 0
assert 0 in state.tool_calls_seen
def test_subsequent_arguments_emit_delta(self):
serving = _serving()
state = _StreamState()
# First call: start.
first = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type("Fn", (), {"name": "calc", "arguments": ""})(),
},
)()
]
serving._handle_tool_call_deltas(state, first)
# Second call: same index, additional arguments fragment.
more = [
type(
"Delta",
(),
{
"index": 0,
"id": None,
"function": type("Fn", (), {"name": None, "arguments": "1}"})(),
},
)()
]
events = serving._handle_tool_call_deltas(state, more)
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-call-delta"
assert payload["index"] == 0
assert (
payload["delta"]["message"]["tool_calls"]["function"]["arguments"] == "1}"
)
def test_new_tool_call_closes_existing_content_block(self):
serving = _serving()
state = _StreamState()
# Open a text block, then start a tool call.
serving._handle_text_delta(state, "I'll call:")
deltas = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type("Fn", (), {"name": "calc", "arguments": "{}"})(),
},
)()
]
events = serving._handle_tool_call_deltas(state, deltas)
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "tool-call-start"]
# ======================================================================
# _handle_citation_deltas
# ======================================================================
class TestHandleCitationDeltas:
"""Test the per-delta emitter. Sources are pre-resolved by the
reasoning parser via the position map forwarded through
``chat_template_kwargs`` (see :func:`_melody_sources_to_vllm` in
``vllm/reasoning/cohere_command_reasoning_parser.py``), so the
handler only has to coerce to the SDK wire shape, apply the
``THINKING_CONTENT`` -> ``PLAN`` rewrite, and drop citations with
no attributable sources.
"""
@staticmethod
def _resolved_source(**overrides: Any) -> CitationSource:
base = dict(
type="document",
id="doc_hamlet",
document={"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
)
base.update(overrides)
return CitationSource(**base)
def test_resolved_citation_emits_start_and_end_with_real_id(self):
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=11,
text="Shakespeare",
sources=[self._resolved_source()],
type="TEXT_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
assert len(events) == 2
start = _parse_event(events[0])
end = _parse_event(events[1])
assert start["type"] == "citation-start"
assert start["index"] == 0
cit_payload = start["delta"]["message"]["citations"]
assert cit_payload["start"] == 0
assert cit_payload["end"] == 11
assert cit_payload["text"] == "Shakespeare"
assert cit_payload["sources"] == [
{
"type": "document",
"id": "doc_hamlet",
"document": {"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
}
]
assert end == {"type": "citation-end", "index": 0}
def test_multiple_sources_survive_to_wire(self):
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="works",
sources=[
self._resolved_source(id="d0", document={"id": "d0"}),
self._resolved_source(id="d1", document={"id": "d1"}),
],
type="TEXT_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
ids = [s["id"] for s in cit_payload["sources"]]
assert ids == ["d0", "d1"]
def test_source_without_id_streamed_with_payload(self):
# Match the cohere api: id-less tool sources (from text-only
# tool results) still ride through with their ``tool_output``
# payload; only the ``id`` field is omitted.
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[CitationSource(type="tool", tool_output={"content": "hi"})],
)
]
events = serving._handle_citation_deltas(state, citations)
assert len(events) >= 1
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
assert cit_payload["sources"] == [
{"type": "tool", "tool_output": {"content": "hi"}}
]
assert "id" not in cit_payload["sources"][0]
def test_citation_with_no_sources_streamed_dropped(self):
# Citations whose source list is entirely empty are still
# dropped so we don't emit an unattributed anchor.
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[],
)
]
events = serving._handle_citation_deltas(state, citations)
assert events == []
def test_plan_rewrite_on_non_reasoning_model(self):
serving = _serving(is_reasoning_model=False)
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[self._resolved_source()],
type="THINKING_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
assert cit_payload["type"] == "PLAN"
# ======================================================================
# _build_message_end_event
# ======================================================================
class TestBuildMessageEndEvent:
def test_without_usage(self):
serving = _serving()
frame = serving._build_message_end_event(chunk_id="abc", finish_reason="stop")
payload = _parse_event(frame)
assert payload["type"] == "message-end"
assert payload["id"] == "abc"
assert payload["delta"]["finish_reason"] == "COMPLETE"
assert "usage" not in payload["delta"]
def test_with_usage(self):
serving = _serving()
chunk = ChatCompletionStreamResponse.model_validate(
{
"id": "abc",
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
}
)
frame = serving._build_message_end_event(
chunk_id="abc", finish_reason="length", usage_chunk=chunk
)
payload = _parse_event(frame)
assert payload["delta"]["finish_reason"] == "MAX_TOKENS"
usage = payload["delta"]["usage"]
assert usage["billed_units"] == {"input_tokens": 10, "output_tokens": 5}
assert usage["tokens"] == {"input_tokens": 10, "output_tokens": 5}
assert "cached_tokens" not in usage
def test_with_cached_tokens(self):
serving = _serving()
chunk = ChatCompletionStreamResponse.model_validate(
{
"id": "abc",
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
"prompt_tokens_details": {"cached_tokens": 3},
},
}
)
frame = serving._build_message_end_event(
chunk_id="abc", finish_reason="stop", usage_chunk=chunk
)
payload = _parse_event(frame)
assert payload["delta"]["usage"]["cached_tokens"] == 3
# ======================================================================
# End-to-end: _chat_completion_stream_to_v2
# ======================================================================
class TestChatCompletionStreamToV2:
"""End-to-end stream lifecycle tests."""
@pytest.mark.asyncio
async def test_text_only_happy_path_emits_full_lifecycle(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
_make_chunk(content=" there"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 4,
"completion_tokens": 2,
"total_tokens": 6,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# message-start, content-start, content-delta, content-delta,
# content-end, message-end, then [DONE] as the last frame.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-delta",
"content-end",
"message-end",
]
assert frames[-1] == _DONE_FRAME
# message-end should carry usage stats from the trailing chunk.
end_payload = _parse_event(frames[-2])
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
@pytest.mark.asyncio
async def test_stop_sequence_finish_reason(self):
serving = _serving()
frames = await _drain(
serving,
[
_make_chunk(role="assistant"),
_make_chunk(
finish_reason="stop",
stop_reason="<END>",
),
_make_chunk(omit_choices=True),
],
)
end_payload = _parse_event(frames[-2])
assert end_payload["delta"]["finish_reason"] == "STOP_SEQUENCE"
@pytest.mark.asyncio
async def test_text_then_tool_call_closes_text_first(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="planning..."),
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "c1",
"type": "function",
"function": {"name": "calc", "arguments": "{}"},
}
]
),
_make_chunk(finish_reason="tool_calls"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# Text block is opened+delta, then closed before the tool-call
# opens, and the final close happens on the usage chunk path.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"tool-call-start",
"tool-call-end",
"message-end",
]
# finish_reason captured from the prior chunk.
end = _parse_event(frames[-2])
assert end["delta"]["finish_reason"] == "TOOL_CALL"
@pytest.mark.asyncio
async def test_thinking_then_text_reasoning_model(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(reasoning="thinking..."),
_make_chunk(content="answer"),
_make_chunk(finish_reason="stop"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# Thinking block opens with index 0; text block reopens with index 1.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"content-start",
"content-delta",
"content-end",
"message-end",
]
thinking_start = _parse_event(frames[1])
assert thinking_start["delta"]["message"]["content"]["type"] == "thinking"
text_start = _parse_event(frames[4])
assert text_start["delta"]["message"]["content"]["type"] == "text"
@pytest.mark.asyncio
async def test_reasoning_on_non_reasoning_model_emits_tool_plan_delta(self):
serving = _serving(is_reasoning_model=False)
items = [
_make_chunk(role="assistant"),
_make_chunk(reasoning="planning"),
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
]
),
_make_chunk(finish_reason="tool_calls"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types == [
"message-start",
"tool-plan-delta",
"tool-call-start",
"tool-call-end",
"message-end",
]
# No thinking content blocks should be present.
assert "content-start" not in types
assert "content-end" not in types
@pytest.mark.asyncio
async def test_stream_exhaustion_closes_open_block_without_usage(self):
# The raw stream may finish without a trailing usage-only chunk.
# The translator must still emit message-end before [DONE] so
# Cohere clients don't hang.
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"message-end",
]
assert frames[-1] == _DONE_FRAME
@pytest.mark.asyncio
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_call"])
async def test_upstream_error_preserves_message(self, block_type):
serving = _serving()
items: list[ChatCompletionStreamResponse | ErrorResponse] = []
if block_type is not None:
items.append(_make_chunk(role="assistant"))
if block_type == "text":
items.append(_make_chunk(content="Hi"))
elif block_type == "thinking":
items.append(_make_chunk(reasoning="Thinking"))
else:
items.append(
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "call_1",
"function": {"name": "search", "arguments": '{"q":'},
}
]
)
)
items.append(serving.create_error_response(ValueError("backend failed")))
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
events = [_parse_event(frame) for frame in frames[:-1]]
error_ends = [event for event in events if event["type"] == "message-end"]
assert len(error_ends) == 1
assert error_ends[0]["delta"] == {
"finish_reason": "ERROR",
"error": "backend failed",
}
assert events[-1] == error_ends[0]
if block_type is not None:
assert events[-2]["type"] == (
"tool-call-end" if block_type == "tool_call" else "content-end"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("started", [False, True])
async def test_error_message_end_is_sanitized(self, started):
# The client-visible ``error`` field on the terminal message-end
# must be routed through ``sanitize_message`` so tracebacks,
# host filesystem paths, and Python object memory addresses
# can't leak from a mid-stream exception into the SSE payload.
serving = _serving()
async def _raises_mid_stream() -> AsyncGenerator[
ChatCompletionStreamResponse | ErrorResponse, None
]:
if started:
yield _make_chunk(role="assistant")
raise RuntimeError(
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
)
frames = [
f async for f in serving._chat_completion_stream_to_v2(_raises_mid_stream())
]
assert frames[-1] == _DONE_FRAME
error_end = _parse_event(frames[-2])
assert error_end["delta"]["finish_reason"] == "ERROR"
err_msg = error_end["delta"]["error"]
# Memory address stripped and path replaced with ``<path>``.
assert "0x7fabc0deface" not in err_msg
assert "/Users/dev/vllm/x.py" not in err_msg
assert "<path>" in err_msg
# The human-facing prefix should still survive sanitization.
assert "boom" in err_msg
@pytest.mark.asyncio
async def test_citations_in_delta_emit_citation_events(self):
# End-to-end streaming citation flow. The parser hands us
# already-resolved sources (see ``_melody_sources_to_vllm``
# in ``vllm/reasoning/cohere_command_reasoning_parser.py``);
# the streaming loop just has to coerce and emit.
serving = _serving()
items = [
_make_chunk(role="assistant"),
_make_chunk(content="hello"),
_make_chunk(
citations=[
{
"start": 0,
"end": 5,
"text": "hello",
"sources": [
{
"type": "document",
"id": "d1",
"document": {"id": "d1", "text": "cited"},
}
],
"type": "TEXT_CONTENT",
}
]
),
_make_chunk(finish_reason="stop"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
parsed = [_parse_event(f) for f in frames[:-1]]
types = [p["type"] for p in parsed]
assert "citation-start" in types
assert "citation-end" in types
cit_start = next(p for p in parsed if p["type"] == "citation-start")
assert cit_start["delta"]["message"]["citations"]["sources"] == [
{
"type": "document",
"id": "d1",
"document": {"id": "d1", "text": "cited"},
}
]
@pytest.mark.asyncio
async def test_first_chunk_emits_message_start_with_chunk_id(self):
serving = _serving()
items = [
_make_chunk(chunk_id="my-id", role="assistant"),
_make_chunk(chunk_id="my-id", content="hi"),
_make_chunk(
chunk_id="my-id",
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
ms = _parse_event(frames[0])
assert ms["type"] == "message-start"
assert ms["id"] == "my-id"
assert ms["delta"]["message"]["role"] == "assistant"
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from pydantic import ValidationError
from vllm.entrypoints.openai.decisions.adapters import (
input_text,
make_answer,
make_read_question,
)
from vllm.entrypoints.openai.decisions.protocol import DecisionRequest
from vllm.entrypoints.openai.decisions.question_types import (
StructuredDecisionError,
)
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
def request(**kwargs):
return DecisionRequest.model_validate(
{
"model": "test-model",
"input": "The package is broken.",
"questions": [{"type": "predicate", "instructions": "Is it damaged?"}],
**kwargs,
}
)
def test_predicate_returns_probability_of_true_and_null_name():
question = request().questions[0]
read = make_read_question(0, question)
assert [option.name for option in read.options] == ["false", "true"]
assert make_answer(question, [0.1, 0.9], 0.5).model_dump() == {
"type": "predicate",
"name": None,
"probability": 0.9,
}
def test_choice_preserves_boolean_and_string_values():
question = request(
questions=[
{
"type": "choice",
"name": "decision",
"instructions": "Select a value.",
"choices": [{"value": True}, {"value": "true"}, {"value": False}],
}
]
).questions[0]
read = make_read_question(0, question)
assert [option.name for option in read.options] == ["true", '"true"', "false"]
answer = make_answer(question, [0.7, 0.2, 0.1], 0.5).model_dump()
assert answer["name"] == "decision"
assert answer["choice"] is True
assert answer["probabilities"] == [
{"value": True, "probability": 0.7},
{"value": "true", "probability": 0.2},
{"value": False, "probability": 0.1},
]
assert answer["confidence"] == pytest.approx(0.35)
def test_score_is_weighted_average_of_zero_based_level_indices():
question = request(
questions=[
{
"type": "score",
"instructions": "Rate severity.",
"levels": [{"label": label} for label in ("low", "medium", "high")],
}
]
).questions[0]
answer = make_answer(question, [0.1, 0.7, 0.2], 0.8).model_dump()
assert answer["score"] == pytest.approx(1.1)
assert answer["probabilities"] == [
{"value": 0, "label": "low", "probability": 0.1},
{"value": 1, "label": "medium", "probability": 0.7},
{"value": 2, "label": "high", "probability": 0.2},
]
assert answer["confidence"] == pytest.approx(0.56)
def test_text_messages_preserve_evidence_order():
parsed = request(
input=[
{"role": "user", "content": "first"},
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "second"},
{"type": "input_text", "text": "third"},
],
},
]
)
assert input_text(parsed.input) == "first\n\nsecond\nthird"
assert input_text([]) == ""
@pytest.mark.parametrize("evidence", ["", [], [{"role": "user", "content": []}]])
def test_empty_input_is_valid_under_the_openai_schema(evidence):
assert input_text(request(input=evidence).input) == ""
def test_score_levels_with_the_same_label_keep_their_positions():
question = request(
questions=[
{
"type": "score",
"instructions": "Rate severity.",
"levels": [
{"label": "damage", "description": "cosmetic"},
{"label": "damage", "description": "unusable"},
],
}
]
).questions[0]
read = make_read_question(0, question)
assert read.labels == ("A", "B")
answer = make_answer(question, [0.1, 0.9], 0.8).model_dump()
assert answer["score"] == pytest.approx(0.9)
assert [p["value"] for p in answer["probabilities"]] == [0, 1]
@pytest.mark.parametrize(
"override,match",
[
({"input": [{"role": "assistant", "content": "x"}]}, "user"),
(
{
"input": [
{
"role": "user",
"content": [
{
"type": "input_image",
"image_url": "data:image/png;base64,AA==",
}
],
}
]
},
"text input only",
),
({"input": {"text": "x"}}, "Input should"),
({"questions": []}, "at least 1"),
(
{"questions": [{"type": "predicate", "instructions": "x", "name": None}]},
"name must be a string",
),
({"questions": [{"type": "predicate", "instructions": 1}]}, "valid string"),
(
{"questions": [{"type": "predicate", "instructions": "x", "criteria": {}}]},
"Extra inputs",
),
({"safety_identifier": "x" * 129}, "at most 128"),
({"stream": True}, "Extra inputs"),
],
)
def test_rejects_invalid_or_unsupported_inputs(override, match):
with pytest.raises(ValidationError, match=match):
request(**override)
@pytest.mark.parametrize("values", [[True, True], ["a", "a"], [1, "a"], [None, "a"]])
def test_choice_rejects_duplicates_and_non_string_non_boolean_values(values):
with pytest.raises(ValidationError):
request(
questions=[
{
"type": "choice",
"instructions": "x",
"choices": [{"value": v} for v in values],
}
]
)
@pytest.mark.parametrize(
"count,valid", [(1, False), (2, True), (255, True), (256, False)]
)
def test_choice_limits(count, valid):
data = {
"questions": [
{
"type": "choice",
"instructions": "x",
"choices": [{"value": str(i)} for i in range(count)],
}
]
}
if valid:
assert len(request(**data).questions[0].choices) == count
else:
with pytest.raises(ValidationError):
request(**data)
@pytest.mark.parametrize(
"count,valid", [(1, False), (2, True), (10, True), (11, False)]
)
def test_score_limits(count, valid):
data = {
"questions": [
{
"type": "score",
"instructions": "x",
"levels": [{"label": str(i)} for i in range(count)],
}
]
}
if valid:
request(**data)
else:
with pytest.raises(ValidationError):
request(**data)
def test_question_limit_and_optional_safety_identifier():
questions = [{"type": "predicate", "instructions": "x"}] * 200
assert len(request(questions=questions, safety_identifier=None).questions) == 200
with pytest.raises(ValidationError, match="at most 200"):
request(questions=[*questions, questions[0]])
def test_backend_rejects_more_choices_than_single_token_labels():
question = request(
questions=[
{
"type": "choice",
"instructions": "x",
"choices": [{"value": str(i)} for i in range(27)],
}
]
).questions[0]
with pytest.raises(StructuredDecisionError, match="at most 26 choices"):
make_read_question(0, question)
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from vllm.entrypoints.openai.decisions.question_types import (
QUESTION_TYPES,
Option,
Question,
QuestionType,
StructuredDecisionError,
build_question,
register_question_type,
)
def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=128):
return build_question(
qid,
"choice",
instructions,
criteria or {"billing": "money", "outage": None, "other": None},
max_options,
)
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
return build_question(qid, "noul", instructions, criteria, 128)
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
return build_question(
qid,
"score",
instructions,
criteria or ["calm", "annoyed", "furious"],
max_options,
)
def test_choice_labels_follow_option_order():
q = choice()
assert q.labels == ("A", "B", "C")
assert [a.name for a in q.options] == ["billing", "outage", "other"]
assert q.options[0].description == "money"
wide = choice(criteria={str(i): None for i in range(26)})
assert wide.labels[-1] == "Z"
def test_choice_prompt():
q = choice()
assert q.type.prompt(q) == (
"Question: Which team?\n"
"A: billing - money\n"
"B: outage\n"
"C: other\n"
"Answer with the letter of one option only."
)
def test_choice_answer_shape():
q = choice()
answer = q.type.answer(q, [0.2, 0.7, 0.1], 0.5)
assert answer == {
"type": "choice",
"choice": "outage",
"probabilities": {"billing": 0.2, "outage": 0.7, "other": 0.1},
"confidence": 0.35,
}
@pytest.mark.parametrize(
"qid,type_name,criteria,match",
[
("", "choice", {"x": None, "y": None}, "non-empty"),
("q", "nope", {"x": None, "y": None}, "unknown question type"),
("q", "choice", ["x", "y"], "must map option names"),
("q", "noul", ["yes", "no"], "must be an object"),
("q", "noul", {"maybe": "same day"}, "must be an object"),
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
("q", "score", ["calm"], "ordered list of levels"),
],
)
def test_build_question_rejects(qid, type_name, criteria, match):
with pytest.raises(StructuredDecisionError, match=match):
build_question(qid, type_name, "", criteria, 128)
def test_one_option_choice():
q = choice(criteria={"only": None})
assert q.labels == ("A",)
assert q.type.answer(q, [1.0], 0.4)["confidence"] == 0.4
def test_option_limit():
with pytest.raises(StructuredDecisionError, match="at most 2 options"):
choice(criteria={"x": None, "y": None, "z": None}, max_options=2)
with pytest.raises(StructuredDecisionError, match="at most 26 options"):
choice(criteria={str(i): None for i in range(27)})
def test_noul_labels_and_prompt():
q = noul()
assert q.labels == ("yes", "no")
assert [a.name for a in q.options] == ["yes", "no"]
assert q.type.prompt(q) == (
"Question: Reply within the hour?\nyes\nno\nAnswer with yes or no only."
)
described = noul(criteria={"true": "same day", "false": "later"})
assert described.type.prompt(described) == (
"Question: Reply within the hour?\n"
"yes: same day\n"
"no: later\n"
"Answer with yes or no only."
)
def test_noul_answer_shape():
q = noul()
answer = q.type.answer(q, [0.8, 0.2], 0.5)
assert answer == {
"type": "noul",
"noul": 0.8,
"probabilities": {"yes": 0.8, "no": 0.2},
"confidence": 0.4,
}
def test_score_labels_are_levels():
assert score().labels == ("0", "1", "2")
with pytest.raises(StructuredDecisionError, match="at most 10"):
score(criteria=[str(i) for i in range(11)])
def test_score_prompt():
q = score()
assert q.type.prompt(q) == (
"Question: How angry?\n"
"0: calm\n"
"1: annoyed\n"
"2: furious\n"
"Answer with the number of one level only."
)
def test_score_answer_shape():
q = score()
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
assert answer == {
"type": "score",
"score": 0.75,
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
"confidence": 0.2,
}
def test_registered_type_plugs_in():
class BinaryQuestion(QuestionType):
name = "test_binary"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
return [Option("yes"), Option("no")]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {"type": self.name, "yes": probs[0]}
register_question_type(BinaryQuestion)
try:
q = build_question("ok", "test_binary", "Is it fine?", None, 128)
assert q.labels == ("A", "B")
assert q.type.answer(q, [0.9, 0.1], 1.0) == {"type": "test_binary", "yes": 0.9}
with pytest.raises(ValueError, match="already registered"):
register_question_type(BinaryQuestion)
finally:
del QUESTION_TYPES["test_binary"]
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
"--max-num-seqs",
"32",
"--enable-prefix-caching",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def post(server, body):
return requests.post(server.url_for("v1/systemone"), json=body)
def choice(instructions, *names):
return {
"type": "choice",
"instructions": instructions,
"criteria": dict.fromkeys(names),
}
COPY = {
"team": choice(
"Which team does the message name?", "billing", "shipping", "security"
),
"lang": choice("Which language does the message name?", "English", "French"),
}
def test_choice_decision(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": {"team": "billing", "language": "French"},
"questions": COPY,
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
assert body["object"] == "structured_decision"
assert list(body["answers"]) == ["team", "lang"]
team = body["answers"]["team"]
assert team["type"] == "choice"
assert set(team["probabilities"]) == {"billing", "shipping", "security"}
assert sum(team["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
assert team["confidence"] == pytest.approx(
max(team["probabilities"].values()) * body["diagnostics"]["team"]["label_mass"]
)
for diag in body["diagnostics"].values():
assert 0.0 < diag["label_mass"] <= 1.0 + 1e-6
assert body["usage"]["output_tokens"] == 2
assert body["usage"]["input_tokens"] > 0
def test_one_option_choice(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": "Team: billing.",
"questions": {
"team": choice("Which team does the message name?", "billing")
},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
team = body["answers"]["team"]
assert team["choice"] == "billing"
assert team["probabilities"] == {"billing": 1.0}
assert team["confidence"] == pytest.approx(
body["diagnostics"]["team"]["label_mass"]
)
def test_one_option_coin_flip(server):
# A fair coin with only "heads" allowed. The answer is forced, so its
# confidence must come from the model's probability of the label.
response = post(
server,
{
"model": MODEL_NAME,
"state": "I flip a fair coin.",
"questions": {"coin": choice("How did the coin land?", "heads")},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
coin = body["answers"]["coin"]
assert coin["probabilities"] == {"heads": 1.0}
assert coin["confidence"] == pytest.approx(
body["diagnostics"]["coin"]["label_mass"]
)
@pytest.mark.parametrize(
"state,questions,expected",
[
(
{"team": "billing", "language": "French"},
COPY,
{"team": "billing", "lang": "French"},
),
(
"My card was charged twice for one order.",
{
"card": choice("Does the message mention a card?", "yes", "no"),
"mood": choice("How does the customer feel?", "happy", "angry"),
},
{"card": "yes", "mood": "angry"},
),
(
"Hello.",
{
"sky": choice(
"What color is the sky on a clear day?", "blue", "green", "red"
)
},
{"sky": "blue"},
),
],
)
def test_single_read_answers(server, state, questions, expected):
response = post(
server,
{
"model": MODEL_NAME,
"state": state,
"questions": questions,
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
answers = response.json()["answers"]
assert {qid: a["choice"] for qid, a in answers.items()} == expected
def test_option_limit(server):
body = {
"model": MODEL_NAME,
"state": "Pick option 7.",
"chat_template_kwargs": {"enable_thinking": False},
}
ok = post(
server, {**body, "questions": {"n": choice("Which?", *map(str, range(26)))}}
)
assert ok.status_code == 200, ok.text
assert len(ok.json()["answers"]["n"]["probabilities"]) == 26
over = post(
server, {**body, "questions": {"n": choice("Which?", *map(str, range(27)))}}
)
assert over.status_code == 400
assert "at most 26 options" in over.json()["error"]["message"]
def test_thinking_is_refused(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": "x",
"questions": {"q": choice("Which?", "a", "b")},
"chat_template_kwargs": {"enable_thinking": True},
},
)
assert response.status_code == 400
assert "thinking must be off" in response.json()["error"]["message"]
def test_noul_and_score(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": {"ticket": "My card was charged twice for one order."},
"questions": {
"urgent": {"type": "noul", "instructions": "Is money moving?"},
"tone": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "furious"],
},
},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
urgent = body["answers"]["urgent"]
assert urgent["type"] == "noul"
assert set(urgent["probabilities"]) == {"yes", "no"}
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
pytest.approx(1.0, abs=1e-4)
)
assert urgent["noul"] == urgent["probabilities"]["yes"]
assert urgent["confidence"] == pytest.approx(
max(urgent["probabilities"].values())
* body["diagnostics"]["urgent"]["label_mass"]
)
tone = body["answers"]["tone"]
assert tone["type"] == "score"
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
assert set(tone["probabilities"]) == {"0", "1", "2"}
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
assert tone["score"] == pytest.approx(
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
)
assert 0 <= tone["score"] <= 2
assert body["usage"]["output_tokens"] == 2
@pytest.mark.parametrize(
"questions,match",
[
(
{
"q": {
"type": "choice",
"criteria": {"a": None, "b": None},
"depends_on": ["x"],
}
},
"unknown field(s)",
),
(
{
f"q{i}": {"type": "choice", "criteria": {"a": None, "b": None}}
for i in range(65)
},
"at most 64",
),
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
],
)
def test_validation_errors(server, questions, match):
response = post(server, {"model": MODEL_NAME, "state": "x", "questions": questions})
assert response.status_code == 400
assert match in response.json()["error"]["message"]
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
import os
import socket
import threading
import time
@@ -410,198 +409,3 @@ def test_external_process_monitoring(api_server_args):
mock_coordinator.shutdown()
time.sleep(0.2)
WORKER_RUNTIME_SECONDS = prev_worker_runtime
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch):
"""The Rust frontend command carries every non-default arg as JSON.
Credentials must not reach the log line."""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
hf_token = "hf_TESTTOKEN123"
api_key = "sk-TESTKEY456"
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--model", "org/model", "--hf-token", hf_token, "--api-key", api_key]
)
class _FakeProc:
pid = 4321
returncode = 0
def poll(self):
return 0
def wait(self, timeout=None):
return 0
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
try:
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
with patch("vllm.v1.utils.logger.info") as log_info:
RustFrontendProcessManager(
binary_path="/nonexistent/vllm-rs",
sock=sock,
args=args,
input_listener=input_listener,
output_listener=output_listener,
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
)
finally:
sock.close()
log_format, *log_args = log_info.call_args.args
message = log_format % tuple(log_args)
assert "Launching Rust frontend:" in message
assert hf_token not in message
assert api_key not in message
assert '"hf_token": "***"' in message
assert '"api_key": "***"' in message
def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path):
"""Config-only model selection must be forwarded to the Rust frontend."""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
config_path = tmp_path / "config.yml"
config_path.write_text("model: org/model\n")
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--config", str(config_path)]
)
assert args.model == "org/model"
assert args.model_tag is None
class _FakeProc:
pid = 4321
returncode = 0
def poll(self):
return 0
def wait(self, timeout=None):
return 0
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
try:
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
with caplog.at_level("INFO", logger="vllm.v1.utils"):
RustFrontendProcessManager(
binary_path="/nonexistent/vllm-rs",
sock=sock,
args=args,
input_listener=input_listener,
output_listener=output_listener,
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
)
finally:
sock.close()
assert '"model_tag": "org/model"' in caplog.text
def test_rust_frontend_inherits_grpc_listener(monkeypatch):
"""The Python-bound gRPC socket is passed to Rust like the HTTP one."""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--model", "org/model", "--grpc-port", "50051"]
)
class _FakeProc:
pid = 4321
returncode = 0
def poll(self):
return 0
def wait(self, timeout=None):
return 0
popen_calls = []
def fake_popen(cmd, **kwargs):
popen_calls.append((cmd, kwargs))
return _FakeProc()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
input_fd = input_listener.socket.fileno()
output_fd = output_listener.socket.fileno()
try:
monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
RustFrontendProcessManager(
binary_path="/nonexistent/vllm-rs",
sock=sock,
args=args,
input_listener=input_listener,
output_listener=output_listener,
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
grpc_sock=grpc_sock,
)
[(cmd, kwargs)] = popen_calls
grpc_fd = grpc_sock.fileno()
flag_index = cmd.index("--grpc-listen-fd")
assert cmd[flag_index + 1] == str(grpc_fd)
assert list(kwargs["pass_fds"]) == [
sock.fileno(),
input_fd,
output_fd,
grpc_fd,
]
finally:
sock.close()
grpc_sock.close()
def test_rust_frontend_spawn_failure_cleans_up_listeners(tmp_path):
"""No child owns the listeners when Popen fails, so the parent removes them."""
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--model", "org/model"]
)
input_listener = make_zmq_listener(f"ipc://{tmp_path}/input.sock", zmq.ROUTER)
output_listener = make_zmq_listener(f"ipc://{tmp_path}/output.sock", zmq.PULL)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
with pytest.raises(FileNotFoundError):
RustFrontendProcessManager(
binary_path=str(tmp_path / "missing-vllm-rs"),
sock=sock,
args=args,
input_listener=input_listener,
output_listener=output_listener,
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
)
finally:
sock.close()
for listener in (input_listener, output_listener):
assert listener.socket.fileno() == -1
assert not os.path.exists(listener.address.removeprefix("ipc://"))
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
import pytest
from starlette.datastructures import State
import vllm.entrypoints.launchers.render.app_state as app_state_mod
from vllm.config import ModelConfig, VllmConfig
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
class _CaptureKwargs:
"""Stands in for OnlineRenderer/OnlineDerenderer; records init kwargs."""
captured: list[dict]
def __init__(self, **kwargs):
type(self).captured.append(kwargs)
def warmup(self):
pass
def _render_cli_args(*argv: str) -> Namespace:
"""Parse ``argv`` the way ``vllm launch render`` does, so new serve flags
carry their defaults instead of needing a hand-maintained ``Namespace``."""
args = make_arg_parser(FlexibleArgumentParser()).parse_args(list(argv))
if args.model_tag is not None:
args.model = args.model_tag
return args
@pytest.mark.asyncio
async def test_render_app_state_uses_config_resolved_reasoning_parser(monkeypatch):
"""gpt_oss defaults its reasoning parser through verify_and_update_config;
the render server must pick up that resolved value like the main API
server does, otherwise harmony markup leaks unparsed into derender output.
"""
model_config = ModelConfig("openai/gpt-oss-20b")
vllm_config = VllmConfig(model_config=model_config)
assert vllm_config.structured_outputs_config.reasoning_parser == "openai_gptoss"
class _Renderer(_CaptureKwargs):
captured = []
class _Derenderer(_CaptureKwargs):
captured = []
async def _noop_async(*args, **kwargs):
return None
monkeypatch.setattr(app_state_mod, "renderer_from_config", lambda cfg: object())
monkeypatch.setattr(app_state_mod, "OnlineRenderer", _Renderer)
monkeypatch.setattr(app_state_mod, "OnlineDerenderer", _Derenderer)
monkeypatch.setattr(app_state_mod, "ServingTokenization", lambda *a, **kw: object())
monkeypatch.setattr(app_state_mod, "init_render_state", lambda *a, **kw: None)
monkeypatch.setattr(app_state_mod, "init_endpoint_plugins_state", _noop_async)
args = _render_cli_args("openai/gpt-oss-20b")
await app_state_mod.init_render_app_state(vllm_config, State(), args)
assert _Renderer.captured[0]["reasoning_parser"] == "openai_gptoss"
assert _Derenderer.captured[0]["reasoning_parser"] == "openai_gptoss"
def test_explicit_reasoning_parser_flag_wins_over_model_default():
"""An explicit --reasoning-parser must survive VllmConfig construction
for models that define their own default (gpt_oss); the default only
applies when the resolved value is empty."""
from vllm.engine.arg_utils import AsyncEngineArgs
engine_args = AsyncEngineArgs(
model="openai/gpt-oss-20b", reasoning_parser="deepseek_r1"
)
model_config = engine_args.create_model_config()
vllm_config = VllmConfig(
model_config=model_config,
structured_outputs_config=engine_args.create_structured_outputs_config(),
)
assert vllm_config.structured_outputs_config.reasoning_parser == "deepseek_r1"
+2 -133
View File
@@ -2,11 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
import pytest
import vllm.entrypoints.launchers.cli_args as cli_args_module
from tests.utils import VLLM_PATH
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.launchers.cli_args import (
@@ -180,73 +178,6 @@ def test_multiple_valid_inputs(serve_parser):
### Tests for serve argument validation that run prior to loading
@pytest.mark.parametrize(
"port, extra_args",
[
(50051, []),
(0, ["--port", "0"]),
(50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]),
],
)
def test_rust_grpc_port_accepts_frontend_launch(
vllm_parser, monkeypatch, port, extra_args
):
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
args = vllm_parser.parse_args(
[
"serve",
"--grpc-port",
str(port),
"--data-parallel-size",
"8",
"--data-parallel-size-local",
"4",
"--data-parallel-start-rank",
"4",
"--data-parallel-hybrid-lb",
*extra_args,
]
)
validate_parsed_serve_args(args)
assert args.grpc_port == port
@pytest.mark.parametrize(
"rust_enabled, extra_args, port",
[
(False, [], 50051),
(False, ["--grpc"], 50051),
(True, ["--grpc"], 50051),
(True, ["--headless"], 50051),
(True, ["--api-server-count", "0"], 50051),
(True, ["--api-server-count", "-1"], 50051),
(True, ["--data-parallel-multi-port-external-lb"], 50051),
(True, ["--port", "50051"], 50051),
(True, [], -1),
(True, [], 65536),
],
)
def test_rust_grpc_port_rejects_incompatible_launch(
vllm_parser, monkeypatch, rust_enabled, extra_args, port
):
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0")
args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args])
error = (
"--grpc and --grpc-port are mutually exclusive"
if "--grpc" in extra_args
else "--grpc-port"
)
with pytest.raises(ValueError, match=error):
validate_parsed_serve_args(args)
def test_rust_grpc_port_requires_serve_subcommand(monkeypatch):
"""Direct Python render/API entrypoints must reject a Rust-only listener."""
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
parser = make_arg_parser(FlexibleArgumentParser())
args = parser.parse_args(["--grpc-port", "50051"])
with pytest.raises(ValueError, match="--grpc-port requires"):
validate_parsed_serve_args(args)
def test_enable_auto_choice_fails_without_tool_call_parser(serve_parser):
@@ -263,7 +194,7 @@ def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
args=[
"--enable-auto-tool-choice",
"--tool-call-parser",
"mistral",
"qwen3_coder",
]
)
validate_parsed_serve_args(args)
@@ -280,7 +211,7 @@ def test_auto_tool_choice_with_reasoning_parser_passes(serve_parser):
args=[
"--enable-auto-tool-choice",
"--tool-call-parser",
"mistral",
"qwen3_coder",
"--reasoning-parser",
"deepseek_r1",
]
@@ -333,48 +264,6 @@ def test_per_request_metrics_requires_log_stats(serve_parser):
validate_parsed_serve_args(args)
def _build_launch_render_parser():
"""Mirror `vllm launch render`.
`vllm/entrypoints/cli/main.py` parses subcommands with ``dest="subparser"``,
and `LaunchSubcommandBase.add_cli_args` builds the component parser with
`make_arg_parser`, so a launch component carries serve args under
``args.subparser == "launch"``.
"""
vllm_parser = FlexibleArgumentParser()
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
launch_parser = subparsers.add_parser("launch")
launch_subparsers = launch_parser.add_subparsers(
required=True, dest="launch_component"
)
render_parser = launch_subparsers.add_parser("render")
make_arg_parser(render_parser)
return vllm_parser
@pytest.fixture
def launch_render_parser():
return _build_launch_render_parser()
@pytest.mark.parametrize(
"cli_args, error",
[
(["--enable-auto-tool-choice"], TypeError),
(["--grpc-port", "50051"], ValueError),
],
)
def test_launch_render_validates_serve_args(
launch_render_parser, monkeypatch, cli_args, error
):
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
args = launch_render_parser.parse_args(args=["launch", "render", *cli_args])
assert args.subparser == "launch"
with pytest.raises(error):
validate_parsed_serve_args(args)
def test_subcommand_without_serve_args_skips_validation():
"""A subcommand that does not use the serve parser is still skipped"""
vllm_parser = FlexibleArgumentParser()
@@ -456,26 +345,6 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
assert args.default_chat_template_kwargs is None
@pytest.mark.parametrize(
"default_kwargs,cohere_format,expected",
[
(None, "cmd3", {"cohere_format": "cmd3"}),
(
{"cohere_format": "cmd4", "custom": True},
"cmd3",
{"cohere_format": "cmd4", "custom": True},
),
],
)
def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
args = SimpleNamespace(
default_chat_template_kwargs=default_kwargs,
cohere_format=cohere_format,
)
assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
def test_default_chat_template_kwargs_invalid_json(serve_parser):
"""Ensure invalid JSON raises an error."""
with pytest.raises(SystemExit):
@@ -1,956 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import subprocess
import tempfile
import threading
from collections.abc import Generator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import vllm.envs as envs
from vllm.assets.audio import AudioAsset
from vllm.connections import HTTPConnection
from vllm.entrypoints.launchers.run_batch import (
BatchRequestOutput,
download_bytes_from_url,
upload_data,
)
from vllm.exceptions import VLLMValidationError
from vllm.utils.mem_constants import MiB_bytes
CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
RERANKER_MODEL_NAME = "BAAI/bge-reranker-v2-m3"
REASONING_MODEL_NAME = "Qwen/Qwen3-0.6B"
SPEECH_LARGE_MODEL_NAME = "openai/whisper-large-v3"
SPEECH_SMALL_MODEL_NAME = "openai/whisper-small"
INPUT_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": CHAT_MODEL_NAME,
"messages": [
{
"role": "system",
"content": "You are a helpful assistant.",
},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": CHAT_MODEL_NAME,
"messages": [
{
"role": "system",
"content": "You are an unhelpful assistant.",
},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
{
"custom_id": "request-3",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": "NonExistModel",
"messages": [
{
"role": "system",
"content": "You are an unhelpful assistant.",
},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
{
"custom_id": "request-4",
"method": "POST",
"url": "/bad_url",
"body": {
"model": CHAT_MODEL_NAME,
"messages": [
{
"role": "system",
"content": "You are an unhelpful assistant.",
},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
{
"custom_id": "request-5",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"stream": "True",
"model": CHAT_MODEL_NAME,
"messages": [
{
"role": "system",
"content": "You are an unhelpful assistant.",
},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
]
)
INVALID_INPUT_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"invalid_field": "request-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": CHAT_MODEL_NAME,
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": CHAT_MODEL_NAME,
"messages": [
{"role": "system", "content": "You are an unhelpful assistant."},
{"role": "user", "content": "Hello world!"},
],
"max_tokens": 1000,
},
},
]
)
INPUT_EMBEDDING_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/embeddings",
"body": {
"model": EMBEDDING_MODEL_NAME,
"input": "You are a helpful assistant.",
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/embeddings",
"body": {
"model": EMBEDDING_MODEL_NAME,
"input": "You are an unhelpful assistant.",
},
},
{
"custom_id": "request-3",
"method": "POST",
"url": "/v1/embeddings",
"body": {
"model": EMBEDDING_MODEL_NAME,
"input": "Hello world!",
},
},
{
"custom_id": "request-4",
"method": "POST",
"url": "/v1/embeddings",
"body": {
"model": "NonExistModel",
"input": "Hello world!",
},
},
]
)
_SCORE_RERANK_DOCUMENTS = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
INPUT_SCORE_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"custom_id": "request-1",
"method": "POST",
"url": "/score",
"body": {
"model": RERANKER_MODEL_NAME,
"queries": "What is the capital of France?",
"documents": _SCORE_RERANK_DOCUMENTS,
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/score",
"body": {
"model": RERANKER_MODEL_NAME,
"queries": "What is the capital of France?",
"documents": _SCORE_RERANK_DOCUMENTS,
},
},
]
)
INPUT_RERANK_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"custom_id": "request-1",
"method": "POST",
"url": "/rerank",
"body": {
"model": RERANKER_MODEL_NAME,
"query": "What is the capital of France?",
"documents": _SCORE_RERANK_DOCUMENTS,
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/rerank",
"body": {
"model": RERANKER_MODEL_NAME,
"query": "What is the capital of France?",
"documents": _SCORE_RERANK_DOCUMENTS,
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v2/rerank",
"body": {
"model": RERANKER_MODEL_NAME,
"query": "What is the capital of France?",
"documents": _SCORE_RERANK_DOCUMENTS,
},
},
]
)
INPUT_REASONING_BATCH = "\n".join(
json.dumps(req)
for req in [
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": REASONING_MODEL_NAME,
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Solve this math problem: 2+2=?"},
],
},
},
{
"custom_id": "request-2",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": REASONING_MODEL_NAME,
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is the capital of France?"},
],
},
},
]
)
MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes
_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3))
_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1)
class _BatchHTTPServer(ThreadingHTTPServer):
request_paths: list[str]
class _BatchHTTPHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
server = self.server
assert isinstance(server, _BatchHTTPServer)
server.request_paths.append(self.path)
if self.path == "/exact-limit":
self._send_body(_EXACT_LIMIT_AUDIO)
return
if self.path == "/oversized-chunked":
self.send_response(200)
self.end_headers()
self._write_body(_OVERSIZED_HTTP_AUDIO)
return
self.send_error(404, "Unknown batch HTTP test path")
def _send_body(self, body: bytes) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self._write_body(body)
def _write_body(self, body: bytes) -> None:
try:
for offset in range(0, len(body), 64 * 1024):
self.wfile.write(body[offset : offset + 64 * 1024])
except (BrokenPipeError, ConnectionResetError):
self.close_connection = True
def log_message(self, fmt: str, *args: object) -> None:
return
@pytest.fixture
def batch_http_server() -> Generator[_BatchHTTPServer, None, None]:
server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler)
server.request_paths = []
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield server
finally:
server.shutdown()
thread.join()
server.server_close()
def _batch_http_url(server: _BatchHTTPServer, path: str) -> str:
return f"http://localhost:{server.server_port}{path}"
async def _close_async_connection(connection: HTTPConnection) -> None:
if connection._async_client is not None:
await connection._async_client.close()
INPUT_TRANSCRIPTION_BATCH = (
json.dumps(
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/audio/transcriptions",
"body": {
"model": SPEECH_LARGE_MODEL_NAME,
"file_url": f"data:audio/wav;base64,{MINIMAL_WAV_BASE64}",
"response_format": "json",
},
}
)
+ "\n"
)
INPUT_TRANSCRIPTION_HTTP_BATCH = (
json.dumps(
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/audio/transcriptions",
"body": {
"model": SPEECH_LARGE_MODEL_NAME,
"file_url": AudioAsset("mary_had_lamb").url,
"language": "en",
"response_format": "json",
},
}
)
+ "\n"
)
INPUT_TRANSLATION_BATCH = (
json.dumps(
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/audio/translations",
"body": {
"model": SPEECH_SMALL_MODEL_NAME,
"file_url": AudioAsset("mary_had_lamb").url,
"response_format": "text",
"language": "it",
"to_language": "en",
"temperature": 0.0,
},
}
)
+ "\n"
)
WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
},
},
"required": ["location"],
},
},
}
INPUT_TOOL_CALLING_BATCH = json.dumps(
{
"custom_id": "request-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": REASONING_MODEL_NAME,
"messages": [
{"role": "user", "content": "What is the weather in San Francisco?"},
],
"tools": [WEATHER_TOOL],
"tool_choice": "required",
"max_tokens": 1000,
},
}
)
def test_empty_file():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write("")
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
EMBEDDING_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
assert contents.strip() == ""
def test_completions():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
CHAT_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
# Ensure that the output format conforms to the openai api.
# Validation should throw if the schema is wrong.
BatchRequestOutput.model_validate_json(line)
def test_completions_invalid_input():
"""Ensure that we fail when the input doesn't conform to the openai api."""
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INVALID_INPUT_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
CHAT_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode != 0, f"{proc=}"
def test_embeddings():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_EMBEDDING_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
EMBEDDING_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
# Ensure that the output format conforms to the openai api.
# Validation should throw if the schema is wrong.
BatchRequestOutput.model_validate_json(line)
@pytest.mark.parametrize("input_batch", [INPUT_SCORE_BATCH, INPUT_RERANK_BATCH])
def test_score(input_batch):
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(input_batch)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
RERANKER_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
# Ensure that the output format conforms to the openai api.
# Validation should throw if the schema is wrong.
BatchRequestOutput.model_validate_json(line)
# Ensure that there is no error in the response.
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
def test_reasoning_parser():
"""Test that reasoning_parser parameter works correctly in run_batch."""
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_REASONING_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
REASONING_MODEL_NAME,
"--reasoning-parser",
"qwen3",
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
# Ensure that the output format conforms to the openai api.
# Validation should throw if the schema is wrong.
BatchRequestOutput.model_validate_json(line)
# Ensure that there is no error in the response.
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
# Check that reasoning is present and not empty
reasoning = line_dict["response"]["body"]["choices"][0]["message"][
"reasoning"
]
assert reasoning is not None
assert len(reasoning) > 0
def test_tool_calling():
"""Test that tool calling works correctly in run_batch.
Verifies that requests with tools return tool_calls in the response.
"""
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_TOOL_CALLING_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
REASONING_MODEL_NAME,
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
if not line.strip(): # Skip empty lines
continue
# Ensure that the output format conforms to the openai api.
# Validation should throw if the schema is wrong.
BatchRequestOutput.model_validate_json(line)
# Ensure that there is no error in the response.
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
# Check that tool_calls are present in the response
# With tool_choice="required", the model must call a tool
response_body = line_dict["response"]["body"]
assert response_body is not None
message = response_body["choices"][0]["message"]
assert "tool_calls" in message
tool_calls = message.get("tool_calls")
# With tool_choice="required", tool_calls must be present and non-empty
assert tool_calls is not None
assert isinstance(tool_calls, list)
assert len(tool_calls) > 0
# Verify tool_calls have the expected structure
for tool_call in tool_calls:
assert "id" in tool_call
assert "type" in tool_call
assert tool_call["type"] == "function"
assert "function" in tool_call
assert "name" in tool_call["function"]
assert "arguments" in tool_call["function"]
# Verify the tool name matches our tool definition
assert tool_call["function"]["name"] == "get_current_weather"
# ---------------------------------------------------------------------------
# Unit tests for download_bytes_from_url SSRF protection
# ---------------------------------------------------------------------------
def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
"""Create mock objects that simulate aiohttp.ClientSession context managers."""
async def iter_chunked(chunk_size: int):
del chunk_size
yield response_data
mock_resp = MagicMock()
mock_resp.status = status
mock_resp.content_length = len(response_data)
mock_resp.content.iter_chunked = iter_chunked
mock_resp.read = AsyncMock(return_value=response_data)
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
mock_resp.__aexit__ = AsyncMock(return_value=False)
mock_session = MagicMock()
mock_session.get = MagicMock(return_value=mock_resp)
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=False)
return mock_session
@pytest.mark.asyncio
async def test_download_bytes_data_url_bypasses_domain_check():
"""data: URLs must work regardless of the domain allowlist."""
data_url = f"data:audio/wav;base64,{MINIMAL_WAV_BASE64}"
result = await download_bytes_from_url(
data_url, allowed_media_domains=["example.com"]
)
assert isinstance(result, bytes)
assert len(result) > 0
@pytest.mark.asyncio
async def test_download_bytes_rejects_disallowed_domain():
"""HTTP URLs whose hostname is not in the allowlist must be rejected."""
url = "https://evil.internal/secret"
with pytest.raises(VLLMValidationError, match="allowed domains") as exc_info:
await download_bytes_from_url(url, allowed_media_domains=["example.com"])
# URL validation failures carry structured metadata for the frontend.
assert exc_info.value.parameter == "url"
assert exc_info.value.value == "evil.internal"
@pytest.mark.asyncio
async def test_download_bytes_rejects_unsupported_scheme():
"""Unsupported URL schemes are rejected with structured metadata."""
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme") as exc_info:
await download_bytes_from_url("ftp://example.com/file")
assert exc_info.value.parameter == "url"
assert exc_info.value.value == "ftp"
@pytest.mark.asyncio
async def test_download_bytes_rejects_cloud_metadata_ip():
"""Cloud metadata endpoints must be blocked when an allowlist is set."""
url = "http://169.254.169.254/latest/meta-data/"
with pytest.raises(VLLMValidationError, match="allowed domains"):
await download_bytes_from_url(url, allowed_media_domains=["example.com"])
@pytest.mark.asyncio
async def test_download_bytes_rejects_internal_ip():
"""Private-range IPs must be blocked when an allowlist is set."""
for internal_url in [
"http://10.0.0.1/secret",
"http://192.168.1.1/admin",
"http://127.0.0.1:8080/internal",
]:
with pytest.raises(VLLMValidationError, match="allowed domains"):
await download_bytes_from_url(
internal_url, allowed_media_domains=["example.com"]
)
@pytest.mark.asyncio
async def test_download_bytes_allows_permitted_domain():
"""HTTP URLs whose hostname IS in the allowlist must be fetched."""
url = "https://example.com/audio.wav"
expected = b"audio-bytes"
mock_session = _make_aiohttp_mocks(expected)
with patch(
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
return_value=mock_session,
):
result = await download_bytes_from_url(
url, allowed_media_domains=["example.com"]
)
assert result == expected
@pytest.mark.asyncio
async def test_download_bytes_no_allowlist_permits_any_domain():
"""Without an allowlist all HTTP URLs must be attempted (backward compat)."""
url = "https://any-domain.example.org/file.wav"
expected = b"some-data"
mock_session = _make_aiohttp_mocks(expected)
with patch(
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
return_value=mock_session,
):
result = await download_bytes_from_url(url, allowed_media_domains=None)
assert result == expected
@pytest.mark.asyncio
async def test_download_bytes_empty_allowlist_denies_all():
"""An empty allowlist must deny all HTTP URLs (least privilege)."""
url = "https://any-domain.example.org/file.wav"
with pytest.raises(VLLMValidationError, match="allowed domains"):
await download_bytes_from_url(url, allowed_media_domains=[])
@pytest.mark.asyncio
async def test_download_bytes_unsupported_scheme():
"""Unsupported URL schemes must be rejected regardless of allowlist."""
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme"):
await download_bytes_from_url("ftp://example.com/file.wav")
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme"):
await download_bytes_from_url(
"ftp://example.com/file.wav",
allowed_media_domains=["example.com"],
)
@pytest.mark.asyncio
async def test_download_bytes_backslash_bypass():
"""Backslash-@ URL confusion must not bypass the allowlist.
urllib3.parse_url() and aiohttp/yarl disagree on backslash-before-@.
The fix normalizes through urllib3 before handing to aiohttp.
"""
bypass_url = "http://allowed.example.com\\@evil.internal/secret"
with pytest.raises(VLLMValidationError, match="allowed domains"):
await download_bytes_from_url(
bypass_url, allowed_media_domains=["evil.internal"]
)
@pytest.mark.asyncio
async def test_download_bytes_allows_http_body_at_audio_limit(
monkeypatch: pytest.MonkeyPatch,
batch_http_server: _BatchHTTPServer,
):
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
connection = HTTPConnection()
monkeypatch.setattr(
"vllm.entrypoints.launchers.run_batch.global_http_connection",
connection,
)
try:
result = await download_bytes_from_url(
_batch_http_url(batch_http_server, "/exact-limit")
)
assert result == _EXACT_LIMIT_AUDIO
finally:
await _close_async_connection(connection)
# ---------------------------------------------------------------------------
# Unit tests for upload_data retry behavior
# ---------------------------------------------------------------------------
def _make_aiohttp_put_session(status: int = 200, body_text: str = ""):
"""Mock an aiohttp.ClientSession whose PUT returns the given status."""
mock_resp = MagicMock()
mock_resp.status = status
mock_resp.text = AsyncMock(return_value=body_text)
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
mock_resp.__aexit__ = AsyncMock(return_value=False)
mock_session = MagicMock()
mock_session.put = MagicMock(return_value=mock_resp)
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=False)
return mock_session
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [200, 201, 204])
@pytest.mark.parametrize("from_file", [False, True])
async def test_upload_data_uploads_once_on_success(status, from_file, tmp_path):
"""Successful PUT responses must not retry either upload path."""
session = _make_aiohttp_put_session(status=status)
data_or_file = "payload"
if from_file:
path = tmp_path / "output.jsonl"
path.write_text(data_or_file, encoding="utf-8")
data_or_file = str(path)
with (
patch(
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
return_value=session,
),
patch(
"vllm.entrypoints.launchers.run_batch.asyncio.sleep",
new_callable=AsyncMock,
) as sleep,
):
await upload_data(
"https://example.com/output.jsonl", data_or_file, from_file=from_file
)
assert session.put.call_count == 1
sleep.assert_not_awaited()
@pytest.mark.asyncio
async def test_upload_data_error_includes_awaited_response_body():
"""A failed upload must surface the awaited response body, not a coroutine."""
session = _make_aiohttp_put_session(status=500, body_text="server-error-detail")
with (
patch(
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
return_value=session,
),
patch(
"vllm.entrypoints.launchers.run_batch.asyncio.sleep",
AsyncMock(),
),
pytest.raises(Exception) as exc_info,
):
await upload_data(
"https://example.com/output.jsonl", "payload", from_file=False
)
message = str(exc_info.value)
assert "server-error-detail" in message
assert "coroutine" not in message
@@ -29,16 +29,6 @@ MODEL_CONFIGS = [
"tensor_parallel_size": 1,
"tokenizer": "Qwen/Qwen3-4B",
},
{
"model": "mistralai/Mistral-7B-Instruct-v0.1",
"enforce_eager": True,
"gpu_memory_utilization": 0.95,
"max_model_len": 64,
"max_num_batched_tokens": 64,
"max_num_seqs": 64,
"tensor_parallel_size": 1,
"tokenizer_mode": "mistral",
},
# TODO: re-enable once these tests are run with V1
# {
# "model": "sentence-transformers/all-MiniLM-L12-v2",
@@ -16,7 +16,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
@@ -196,58 +195,6 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
)
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_render.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_render
@pytest.mark.asyncio
async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_render = _build_serving_render(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_render.render_chat_request(request)
assert result.token_ids == [1, 2, 3]
assert (
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_chat_error_stream(include_usage: bool):
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
"--max-num-seqs",
"32",
"--enable-prefix-caching",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def test_openai_decisions_text_questions(server):
response = requests.post(
server.url_for("v1/decisions"),
json={
"model": MODEL_NAME,
"input": "The package arrived with a broken screen.",
"questions": [
{"type": "predicate", "instructions": "Is the screen broken?"},
{
"type": "choice",
"name": "department",
"instructions": "Which department should handle this?",
"choices": [{"value": "returns"}, {"value": "billing"}],
},
{
"type": "score",
"name": "damage",
"instructions": "Rate the damage.",
"levels": [{"label": "undamaged"}, {"label": "broken"}],
},
],
},
)
assert response.status_code == 200, response.text
body = response.json()
assert set(body) == {"model", "answers", "usage"}
predicate, choice_answer, score = body["answers"]
assert predicate["type"] == "predicate"
assert predicate["name"] is None
assert 0 <= predicate["probability"] <= 1
assert choice_answer["type"] == "choice"
assert choice_answer["name"] == "department"
assert [p["value"] for p in choice_answer["probabilities"]] == [
"returns",
"billing",
]
assert (
choice_answer["choice"]
== max(choice_answer["probabilities"], key=lambda p: p["probability"])["value"]
)
assert sum(
p["probability"] for p in choice_answer["probabilities"]
) == pytest.approx(1)
assert score["type"] == "score"
assert score["name"] == "damage"
assert [p["label"] for p in score["probabilities"]] == ["undamaged", "broken"]
assert score["score"] == pytest.approx(
sum(p["value"] * p["probability"] for p in score["probabilities"])
)
usage = body["usage"]
assert usage["output_tokens"] == 3
assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"]
def test_openai_decisions_rejects_images(server):
response = requests.post(
server.url_for("v1/decisions"),
json={
"model": MODEL_NAME,
"input": [
{
"role": "user",
"content": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
],
"questions": [{"type": "predicate", "instructions": "Is this red?"}],
},
)
assert response.status_code == 400
assert "text input only" in response.json()["error"]["message"]
@@ -2,9 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token-offsets request/response protocol wiring:
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
serialization boundary. End-to-end behavior is covered by
``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
storage is not retested here.
serialization boundary. Plain Pydantic field storage is not retested here.
"""
from unittest.mock import Mock
View File
-349
View File
@@ -1,349 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared fixtures and helpers for the RL lifecycle test suite.
All test modules under this directory import from here to avoid duplication.
RFC: https://github.com/vllm-project/vllm/issues/45585
PR: https://github.com/vllm-project/vllm/pull/45586
"""
import contextlib
import json
import os
import subprocess
import sys
import threading
import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import Any
import requests
# ---------------------------------------------------------------------------
# Model / server defaults
# ---------------------------------------------------------------------------
MODEL_NAME = os.environ.get("VLLM_TEST_MODEL", "Qwen/Qwen3-0.6B")
_BASE_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"32",
"--gpu-memory-utilization",
"0.75",
"--enable-sleep-mode",
"--enforce-eager",
]
# Lightweight args for state-machine / protocol tests that don't need real
# weights (avoids spending time downloading a 1B checkpoint in T0 tests).
_DUMMY_ARGS = [
"--dtype",
"bfloat16",
"--max-model-len",
"128",
"--max-num-seqs",
"8",
"--gpu-memory-utilization",
"0.5",
"--enable-sleep-mode",
"--enforce-eager",
"--load-format",
"dummy",
]
# ---------------------------------------------------------------------------
# Server harness
# ---------------------------------------------------------------------------
def _warm_up(url: str) -> None:
"""Put one request through the engine before tests start timing things.
/health turns green before any request has travelled the request path, and
that first pass costs seconds on a loaded machine.
"""
response = gen(url, max_tokens=4, timeout=120)
assert ok(response), f"warm-up generation failed: {response}"
@contextmanager
def server(
extra_args=None,
port: int = 8770,
timeout: float = 180.0,
dummy_weights: bool = False,
):
"""Launch a vLLM server with the dev router; yield its base URL.
Args:
extra_args: Additional CLI flags appended after the base args.
port: HTTP port to bind (caller is responsible for uniqueness).
timeout: Seconds to wait for /health before giving up.
dummy_weights: If True, use --load-format dummy (fast, no real weights).
"""
env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"}
base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS
cmd = [
sys.executable,
"-m",
"vllm.entrypoints.openai.api_server",
"--model",
MODEL_NAME,
"--port",
str(port),
"--served-model-name",
"m",
*(base + (extra_args or [])),
]
proc = subprocess.Popen(
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
)
url = f"http://localhost:{port}"
try:
deadline = time.time() + timeout
while time.time() < deadline:
if proc.poll() is not None:
err = (
proc.stderr.read(4000).decode(errors="replace")
if proc.stderr
else ""
)
raise RuntimeError(f"vllm server exited during startup:\n{err}")
with contextlib.suppress(Exception):
if requests.get(f"{url}/health", timeout=3).status_code == 200:
break
time.sleep(1)
else:
proc.terminate()
raise RuntimeError("vllm server did not start in time")
_warm_up(url)
yield url
finally:
proc.terminate()
with contextlib.suppress(subprocess.TimeoutExpired):
proc.wait(timeout=10)
if proc.poll() is None:
proc.kill()
# ---------------------------------------------------------------------------
# HTTP helpers — generation
# ---------------------------------------------------------------------------
def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30):
"""Fire a /v1/completions request; return JSON or None on any error."""
try:
r = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": max_tokens,
"temperature": 0,
},
timeout=timeout,
)
return r.json()
except Exception:
return None
def ok(resp) -> bool:
"""True iff resp is a successful completion (has choices, no error key)."""
return (
resp is not None
and "choices" in resp
and bool(resp["choices"])
and "error" not in resp
)
# ---------------------------------------------------------------------------
# HTTP helpers — stream generation
# ---------------------------------------------------------------------------
# First-token wait for a streaming request; loaded machines need the slack.
STREAM_START_TIMEOUT = 20.0
@dataclass
class StreamResult:
started: threading.Event = field(default_factory=threading.Event)
done: threading.Event = field(default_factory=threading.Event)
chunks: list[dict[str, Any]] = field(default_factory=list)
finish_reason: str | None = None
error: Exception | None = None
def stream_completion(url: str, result: StreamResult, max_tokens: int) -> None:
try:
with requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": "Count upward slowly: one, two, three,",
"max_tokens": max_tokens,
"temperature": 0,
"ignore_eos": True,
"stream": True,
},
stream=True,
timeout=(5, 60),
) as response:
response.raise_for_status()
for line in response.iter_lines(decode_unicode=True):
if not line or line == "data: [DONE]":
continue
assert line.startswith("data: ")
chunk = json.loads(line.removeprefix("data: "))
result.chunks.append(chunk)
choice = chunk["choices"][0]
if choice.get("text"):
result.started.set()
if choice.get("finish_reason") is not None:
result.finish_reason = choice["finish_reason"]
except Exception as error:
result.error = error
finally:
result.done.set()
def start_stream(url: str, max_tokens: int) -> tuple[StreamResult, threading.Thread]:
result = StreamResult()
thread = threading.Thread(
target=stream_completion,
args=(url, result, max_tokens),
)
thread.start()
started = result.started.wait(timeout=STREAM_START_TIMEOUT)
if not started or result.done.is_set():
# Best-effort: on a stalled server these time out too, and would then
# mask the assertions below.
with contextlib.suppress(requests.RequestException):
pause(url, mode="abort")
resume(url)
thread.join(timeout=10)
assert started, (
f"request did not start generating within {STREAM_START_TIMEOUT}s "
f"(stream error: {result.error})"
)
assert not result.done.is_set(), "request completed before it could be paused"
return result, thread
# ---------------------------------------------------------------------------
# HTTP helpers — pause / resume
# ---------------------------------------------------------------------------
def pause(url, mode="abort", clear_cache=True):
return requests.post(
f"{url}/pause",
params={"mode": mode, "clear_cache": clear_cache},
timeout=15,
).status_code
def resume(url):
return requests.post(f"{url}/resume", timeout=10).status_code
def completion_with_cache_details(url: str, prompt: str) -> dict[str, Any]:
response = requests.post(
f"{url}/v1/completions",
json={
"model": "m",
"prompt": prompt,
"max_tokens": 8,
"temperature": 0,
"logprobs": 1,
},
timeout=30,
)
response.raise_for_status()
return response.json()
def golden_output(response: dict[str, Any]) -> dict[str, Any]:
choice = response["choices"][0]
usage = response["usage"]
return {
"text": choice["text"],
"finish_reason": choice["finish_reason"],
"tokens": choice["logprobs"]["tokens"],
"prompt_tokens": usage["prompt_tokens"],
"completion_tokens": usage["completion_tokens"],
}
def cached_tokens(response: dict[str, Any]) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
# ---------------------------------------------------------------------------
# HTTP helpers — sleep / wake
# ---------------------------------------------------------------------------
def sleep(url, level=1, mode="abort"):
return requests.post(
f"{url}/sleep", params={"level": level, "mode": mode}, timeout=15
).status_code
def wake(url, tags=None):
params = {"tags": tags} if tags else {}
return requests.post(f"{url}/wake_up", params=params, timeout=20).status_code
def is_sleeping(url) -> bool:
return requests.get(f"{url}/is_sleeping", timeout=5).json()["is_sleeping"]
def is_paused(url) -> bool:
return requests.get(f"{url}/is_paused", timeout=5).json()["is_paused"]
def health(url) -> int:
try:
return requests.get(f"{url}/health", timeout=5).status_code
except Exception:
return 0
# ---------------------------------------------------------------------------
# HTTP helpers — weight transfer
# ---------------------------------------------------------------------------
def start_weight_update(url, is_checkpoint_format=True):
return requests.post(
f"{url}/start_weight_update",
json={"is_checkpoint_format": is_checkpoint_format},
timeout=10,
)
def finish_weight_update(url):
return requests.post(f"{url}/finish_weight_update", timeout=10)
def get_world_size(url, include_dp=True):
return requests.get(
f"{url}/get_world_size",
params={"include_dp": include_dp},
timeout=5,
)
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import numpy as np
import pybase64 as base64
import pytest
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# Its published sliding window is incompatible with AuxOutput retention.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
def decode_routed_experts(encoded: str | None) -> np.ndarray:
assert encoded is not None
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
return routed_experts
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"256",
"--max-num-seqs",
"32",
"--enforce-eager",
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_routed_experts(server):
"""Test that /v1/completions returns routed_experts when enabled."""
async with server.get_async_client() as client:
result = await client.completions.create(
model=MODEL_NAME,
prompt="Hello, world",
max_tokens=10,
temperature=0,
extra_body={"return_token_ids": True},
)
choice = result.model_dump()["choices"][0]
assert choice["token_ids"] is not None
assert result.usage is not None
routed_experts = decode_routed_experts(choice["routed_experts"])
assert len(routed_experts) == (
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Malformed bodies on RL dev routes are 400s that never reach the engine."""
import json
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.rl.online.api_router import router as rl_router
from vllm.entrypoints.serve.dev.rpc.api_router import router as rpc_router
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
pytestmark = pytest.mark.cpu_test
class Engine:
def __init__(self):
self.calls: list[str] = []
def __getattr__(self, name):
async def call(*args, **kwargs):
self.calls.append(name)
return call
@pytest.fixture
def client():
app = FastAPI()
app.include_router(rl_router)
app.include_router(rpc_router)
init_exception_handler(app)
app.state.engine_client = Engine()
app.state.args = SimpleNamespace(log_error_stack=False)
return TestClient(app)
def post(client, path, body):
return client.post(
path, content=json.dumps(body), headers={"content-type": "application/json"}
)
@pytest.mark.parametrize(
"path",
[
"/abort_requests",
"/init_weight_transfer_engine",
"/update_weights",
"/collective_rpc",
],
)
@pytest.mark.parametrize("body", [[], 1, "x", None])
def test_non_object_body_is_rejected(client, path, body):
assert post(client, path, body).status_code == 400
assert client.app.state.engine_client.calls == []
@pytest.mark.parametrize(
"path,body",
[
("/init_weight_transfer_engine", {}),
("/init_weight_transfer_engine", {"init_info": []}),
("/init_weight_transfer_engine", {"init_info": 1}),
("/update_weights", {}),
("/update_weights", {"update_info": 1}),
("/update_weights", {"update_info": [1, 2]}),
("/update_weights", {"update_info": "x"}),
],
)
def test_invalid_field_is_rejected_before_the_engine(client, path, body):
# Reaching the engine would also abort an in-progress weight update.
assert post(client, path, body).status_code == 400
assert client.app.state.engine_client.calls == []
@pytest.mark.parametrize(
"path,body,engine_call",
[
("/abort_requests", {"request_ids": ["a"]}, "abort"),
(
"/init_weight_transfer_engine",
{"init_info": {}},
"init_weight_transfer_engine",
),
("/update_weights", {"update_info": {"names": []}}, "update_weights"),
("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
("/collective_rpc", {"method": "m"}, "collective_rpc"),
(
"/shutdown_weight_transfer_engine",
{},
"shutdown_weight_transfer_engine",
),
],
)
def test_valid_body_reaches_the_engine(client, path, body, engine_call):
assert post(client, path, body).status_code == 200
assert client.app.state.engine_client.calls == [engine_call]
@@ -1,88 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
GEN_ENDPOINT = "/inference/v1/generate"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# Its published sliding window is incompatible with AuxOutput retention.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"256",
"--max-num-seqs",
"32",
"--enforce-eager",
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
"--enable-scale-out",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={
"VLLM_USE_V2_MODEL_RUNNER": "1",
},
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_routed_experts(client):
"""Test that /inference/v1/generate returns routed_experts when enabled."""
prompt_token_ids = [1, 2, 3]
payload = {
"model": MODEL_NAME,
"token_ids": prompt_token_ids,
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["routed_experts"] is not None
assert choice["token_ids"] is not None
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
@@ -1,168 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the vLLM RL pause/resume lifecycle."""
import os
import threading
from typing import Any
from unittest.mock import patch
import pytest
import requests
from tests.entrypoints.rl.conftest import (
cached_tokens,
completion_with_cache_details,
gen,
golden_output,
is_paused,
ok,
pause,
resume,
server,
start_stream,
)
@pytest.fixture(scope="module", params=[False, True], ids=["MRV1", "MRV2"])
def use_v2(request):
return request.param
@pytest.fixture(scope="module")
def server_url(use_v2):
env_vars = {
"VLLM_USE_V2_MODEL_RUNNER": "1" if use_v2 else "0",
}
with (
patch.dict(os.environ, env_vars),
server(
extra_args=[
"--enable-prefix-caching",
"--enable-prompt-tokens-details",
]
) as url,
):
yield url
@pytest.fixture(autouse=True)
def restore_unpaused_state(server_url):
assert resume(server_url) == 200
yield
assert resume(server_url) == 200
class TestPauseResume:
def test_state_and_idempotency_across_cycles(self, server_url):
assert not is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
for _ in range(2):
for mode in ("abort", "wait", "keep"):
assert pause(server_url, mode=mode) == 200
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
assert resume(server_url) == 200
assert resume(server_url) == 200
assert not is_paused(server_url)
def test_invalid_mode_preserves_state(self, server_url):
for paused in (False, True):
if paused:
assert pause(server_url) == 200
assert is_paused(server_url) is paused
response = requests.post(
f"{server_url}/pause",
params={"mode": "invalid"},
timeout=10,
)
assert response.status_code == 400
assert response.json()["error"]["param"] == "query.mode"
assert is_paused(server_url) is paused
assert resume(server_url) == 200
@pytest.mark.parametrize(
("mode", "max_tokens", "inflight_finish_reason"),
[
pytest.param("abort", 256, "abort", id="abort"),
pytest.param("wait", 256, "length", id="wait"),
pytest.param("keep", 256, "length", id="keep"),
],
)
def test_mode_request_lifecycle(
self,
server_url,
mode,
max_tokens,
inflight_finish_reason,
):
inflight, inflight_thread = start_stream(server_url, max_tokens)
new_result: dict[str, Any] = {}
new_done = threading.Event()
def _new_request():
new_result["response"] = gen(server_url, max_tokens=4, timeout=60)
new_done.set()
new_thread = threading.Thread(target=_new_request)
try:
assert pause(server_url, mode=mode) == 200
assert is_paused(server_url)
if mode in ("abort", "wait"):
assert inflight.done.is_set()
else:
chunks_after_pause = len(inflight.chunks)
assert not inflight.done.wait(timeout=5)
assert len(inflight.chunks) == chunks_after_pause, (
"in-flight request continued generating in keep mode"
)
new_thread.start()
assert not new_done.wait(timeout=0.3), (
"new request completed while generation was paused"
)
finally:
assert resume(server_url) == 200
inflight_thread.join(timeout=30)
if new_thread.ident is not None:
new_thread.join(timeout=30)
assert not inflight_thread.is_alive()
assert inflight.error is None
assert inflight.finish_reason == inflight_finish_reason
assert not new_thread.is_alive()
assert ok(new_result.get("response"))
def test_clear_cache_preserves_output_and_controls_prefix_cache(self, server_url):
prompt = (
"Paris is the capital of France. Berlin is the capital of Germany. "
) * 20
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
baseline = completion_with_cache_details(server_url, prompt)
warmed = completion_with_cache_details(server_url, prompt)
assert cached_tokens(baseline) == 0
assert cached_tokens(warmed) > 0
assert pause(server_url, clear_cache=False) == 200
assert resume(server_url) == 200
preserved = completion_with_cache_details(server_url, prompt)
assert golden_output(preserved) == golden_output(baseline)
assert cached_tokens(preserved) > 0
assert pause(server_url, clear_cache=True) == 200
assert resume(server_url) == 200
cleared = completion_with_cache_details(server_url, prompt)
assert golden_output(cleared) == golden_output(baseline)
assert cached_tokens(cleared) == 0
@@ -1,306 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.instrumentator.metrics import (
attach_router as attach_metrics_router,
)
from vllm.v1.metrics import prometheus as prometheus_metrics
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.fixture
def sleep_route_app(monkeypatch):
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
metrics = SleepModeOperationMetrics(CollectorRegistry())
monkeypatch.setattr(
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
lambda: metrics,
)
attach_router(app)
init_exception_handler(app)
return app, metrics
@pytest.mark.cpu_test
@pytest.mark.parametrize("level", [0, 1, 2])
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
app, _ = sleep_route_app
with TestClient(app) as client:
response = client.post("/sleep", params={"level": level})
assert response.status_code == 200
assert response.json() == {"status": "sleeping", "level": level}
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("query", "expected_param"),
[
("level=invalid", "query.level"),
("level=-1", "query.level"),
("level=3", "query.level"),
("mode=invalid", "query.mode"),
],
)
def test_sleep_route_rejects_invalid_query_before_dispatch(
sleep_route_app, query, expected_param
):
app, metrics = sleep_route_app
with TestClient(app) as client:
response = client.post(f"/sleep?{query}")
assert response.status_code == 400
assert response.json()["error"]["param"] == expected_param
app.state.engine_client.sleep.assert_not_awaited()
assert list(metrics.duration.collect()[0].samples) == []
assert list(metrics.in_flight.collect()[0].samples) == []
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("fully_awake", "tags", "expected"),
[
(True, "", {"status": "awake", "tags": None}),
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
],
)
def test_wake_route_maps_engine_result_without_state_query(
sleep_route_app, fully_awake, tags, expected
):
app, _ = sleep_route_app
app.state.engine_client.wake_up.return_value = fully_awake
with TestClient(app) as client:
response = client.post(f"/wake_up{tags}")
assert response.status_code == 200
assert response.json() == expected
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
app.state.engine_client.is_sleeping.assert_not_awaited()
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(sleep_route_app, fails):
app, metrics = sleep_route_app
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
if not fails:
assert response.json() == {"status": "kv_cache_released"}
release.assert_awaited_once_with()
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
assert _duration_count(metrics, "release_kv_cache_memory") == 1
@pytest.mark.cpu_test
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
metrics = SleepModeOperationMetrics(CollectorRegistry())
with metrics.record(operation):
assert metrics.in_flight.labels(operation)._value.get() == 1
with pytest.raises(exception), metrics.record(operation):
raise exception()
assert metrics.in_flight.labels(operation)._value.get() == 0
assert metrics.duration.labels(operation)._sum.get() >= 0
assert _duration_count(metrics, operation) == 2
def _duration_count(metrics, operation):
return next(
sample.value
for sample in metrics.duration.collect()[0].samples
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("path", "method", "operation", "status", "fails"),
[
("/sleep", "POST", "sleep", 200, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
("/wake_up", "POST", "wake", 200, False),
("/is_sleeping", "GET", None, 200, False),
("/sleep?level=3", "POST", None, 400, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
],
)
def test_sleep_routes_visible_on_production_metrics_endpoint(
monkeypatch, path, method, operation, status, fails
):
registry = CollectorRegistry()
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
monkeypatch.setattr(sleep_metrics, "_metrics", None)
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
app.state.engine_client.wake_up.return_value = True
app.state.engine_client.is_sleeping.return_value = False
if fails:
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
"engine failed"
)
attach_router(app)
init_exception_handler(app)
attach_metrics_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
assert client.request(method, path).status_code == status
response = client.get("/metrics")
assert response.status_code == 200
samples = [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
]
assert any(
sample.name == "http_requests_total"
and sample.labels
== {
"handler": path.split("?")[0],
"method": method,
"status": f"{status // 100}xx",
}
and sample.value == 1
for sample in samples
)
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
assert any(
sample.name == "vllm:sleep_mode_operations_in_flight"
and sample.labels == {"operation": operation}
and sample.value == 0
for sample in samples
) == (operation is not None)
assert any(
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
and sample.labels == {"operation": operation}
and sample.value == 1
for sample in samples
) == (operation is not None)
def test_sleep_mode():
# dtype, max-len etc set so that this can run in CI
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--max-num-seqs",
"128",
"--enable-sleep-mode",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
) as remote_server:
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 0
assert weights_offloaded == 1
assert discard_all == 0
response = requests.post(remote_server.url_for("wake_up"))
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
# test wake up with tags
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
assert response.status_code == 200
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["weights"]}
)
assert response.status_code == 200
# Partial wake keeps the engine sleeping.
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is True
response = requests.post(
remote_server.url_for("wake_up"), params={"tags": ["kv_cache"]}
)
assert response.status_code == 200
response = requests.get(remote_server.url_for("is_sleeping"))
assert response.status_code == 200
assert response.json().get("is_sleeping") is False
# check sleep metrics
response = requests.get(remote_server.url_for("metrics"))
assert response.status_code == 200
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
assert awake == 1
assert weights_offloaded == 0
assert discard_all == 0
def _get_sleep_metrics_from_api(response: requests.Response):
"""Return (awake, weights_offloaded, discard_all)."""
awake, weights_offloaded, discard_all = None, None, None
for family in text_string_to_metric_families(response.text):
if family.name == "vllm:engine_sleep_state":
for sample in family.samples:
if sample.name == "vllm:engine_sleep_state":
for label_name, label_value in sample.labels.items():
if label_value == "awake":
awake = sample.value
elif label_value == "weights_offloaded":
weights_offloaded = sample.value
elif label_value == "discard_all":
discard_all = sample.value
assert awake is not None
assert weights_offloaded is not None
assert discard_all is not None
return awake, weights_offloaded, discard_all
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for weight transfer APIs via LLM class.
These tests use a mock weight transfer engine to verify that the API
calls the correct methods with the right arguments, without requiring
actual NCCL communication.
"""
import os
import weakref
from dataclasses import dataclass
from unittest.mock import patch
import pytest
import torch
from tests.utils import create_new_process_for_each_test
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferInitRequest,
WeightTransferUpdateInfo,
WeightTransferUpdateRequest,
)
# Use a tiny model for fast testing
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
# --- Mock Weight Transfer Engine ---
@dataclass
class MockInitInfo(WeightTransferInitInfo):
"""Mock initialization info."""
test_param: str = "test"
@dataclass
class MockUpdateInfo(WeightTransferUpdateInfo):
"""Mock update info."""
names: list[str] | None = None
dtype_names: list[str] | None = None
shapes: list[list[int]] | None = None
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
"""Mock weight transfer engine that tracks method calls."""
init_info_cls = MockInitInfo
update_info_cls = MockUpdateInfo
# Class-level tracking for verification across processes
init_transfer_engine_called: bool = False
start_called: bool = False
receive_weights_called: bool = False
finish_called: bool = False
shutdown_called: bool = False
last_init_info: MockInitInfo | None = None
last_update_info: MockUpdateInfo | None = None
def __init__(self, config, vllm_config, device, model):
super().__init__(config, vllm_config, device, model)
# Reset tracking on init
MockWeightTransferEngine.init_transfer_engine_called = False
MockWeightTransferEngine.start_called = False
MockWeightTransferEngine.receive_weights_called = False
MockWeightTransferEngine.finish_called = False
MockWeightTransferEngine.shutdown_called = False
MockWeightTransferEngine.last_init_info = None
MockWeightTransferEngine.last_update_info = None
def init_transfer_engine(self, init_info: MockInitInfo) -> None:
MockWeightTransferEngine.init_transfer_engine_called = True
MockWeightTransferEngine.last_init_info = init_info
def start_weight_update(self) -> None:
MockWeightTransferEngine.start_called = True
def finish_weight_update(self) -> None:
MockWeightTransferEngine.finish_called = True
def receive_weights(self, update_info: MockUpdateInfo) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
def shutdown(self) -> None:
MockWeightTransferEngine.shutdown_called = True
def mock_create_engine(config, vllm_config, device, model):
"""Mock factory function that returns our mock engine."""
return MockWeightTransferEngine(config, vllm_config, device, model)
# --- Tests ---
@create_new_process_for_each_test()
def test_get_world_size_tp1(vllm_runner):
"""Test world_size is correctly configured for TP=1."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
with vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
) as runner:
world_size = runner.llm.llm_engine.vllm_config.parallel_config.world_size
assert world_size == 1
@create_new_process_for_each_test()
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
"""Test that init_weight_transfer_engine calls the engine's
init_transfer_engine method."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Run in-process so mock.patch works (spawn won't inherit the mock)
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
) as runner,
):
llm = weakref.proxy(runner.llm)
# Verify engine was created
def check_engine_exists(self):
return self.weight_transfer_engine is not None
results = llm.collective_rpc(check_engine_exists)
assert all(results), "Weight transfer engine should be initialized"
# Call init_weight_transfer_engine
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "hello"})
)
# Verify init_transfer_engine was called on the engine
def check_init_called(self):
engine = self.weight_transfer_engine
return (
engine.init_transfer_engine_called,
engine.last_init_info.test_param if engine.last_init_info else None,
)
results = llm.collective_rpc(check_init_called)
for called, param in results:
assert called, "init_transfer_engine should have been called"
assert param == "hello", f"Expected 'hello', got {param}"
llm.shutdown_weight_transfer_engine()
assert all(
llm.collective_rpc(
lambda worker: worker.weight_transfer_engine.shutdown_called
)
)
@create_new_process_for_each_test()
def test_update_weights_calls_engine(vllm_runner):
"""Test that update_weights calls the engine's receive_weights method."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Run in-process so mock.patch works (spawn won't inherit the mock)
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
) as runner,
):
llm = weakref.proxy(runner.llm)
# First init the weight transfer
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update()
# Call update_weights
test_names = ["layer.weight", "layer.bias"]
test_dtypes = ["float32", "float32"]
test_shapes = [[10, 10], [10]]
llm.update_weights(
WeightTransferUpdateRequest(
update_info={
"names": test_names,
"dtype_names": test_dtypes,
"shapes": test_shapes,
}
)
)
# Verify receive_weights was called with correct info
def check_update_called(self):
engine = self.weight_transfer_engine
if not engine.receive_weights_called:
return False, None, None, None
info = engine.last_update_info
return (True, info.names, info.dtype_names, info.shapes)
results = llm.collective_rpc(check_update_called)
for called, names, dtypes, shapes in results:
assert called, "receive_weights should have been called"
assert names == test_names
assert dtypes == test_dtypes
assert shapes == test_shapes
llm.finish_weight_update()
assert llm.get_weight_version() == "default"
@create_new_process_for_each_test()
def test_full_weight_transfer_flow(vllm_runner):
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Run in-process so mock.patch works (spawn won't inherit the mock)
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
) as runner,
):
llm = weakref.proxy(runner.llm)
assert llm.get_weight_version() == "default"
# Step 1: Initialize weight transfer engine
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "flow_test"})
)
# Step 2: Start weight update
llm.start_weight_update()
# Step 3: Update weights
llm.update_weights(
WeightTransferUpdateRequest(
update_info={
"names": ["test.weight"],
"dtype_names": ["bfloat16"],
"shapes": [[100, 100]],
}
)
)
assert llm.get_weight_version() == "default"
# Step 4: Finish weight update
llm.finish_weight_update("step-42")
assert llm.get_weight_version() == "step-42"
llm.update_weight_version("manual-version")
assert llm.get_weight_version() == "manual-version"
# Verify the full flow completed
def check_flow(self):
engine = self.weight_transfer_engine
return {
"init_called": engine.init_transfer_engine_called,
"update_called": engine.receive_weights_called,
"init_param": (
engine.last_init_info.test_param if engine.last_init_info else None
),
"update_names": (
engine.last_update_info.names if engine.last_update_info else None
),
}
results = llm.collective_rpc(check_flow)
for result in results:
assert result["init_called"], "init_transfer_engine should be called"
assert result["update_called"], "receive_weights should be called"
assert result["init_param"] == "flow_test"
assert result["update_names"] == ["test.weight"]
@create_new_process_for_each_test()
def test_weight_transfer_config_backend(vllm_runner):
"""Test that WeightTransferConfig backend is properly configured."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Test with nccl backend
with vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
) as runner:
config = runner.llm.llm_engine.vllm_config.weight_transfer_config
assert config is not None
assert config.backend == "nccl"
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import os
import subprocess
import sys
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from vllm.entrypoints.rl.online import api_router
from vllm.entrypoints.rl.online import metrics as rl_metrics
from vllm.entrypoints.rl.online.metrics import WeightOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.v1.metrics.prometheus import get_prometheus_registry
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
DURATION = "vllm:rl_weight_update_operation_duration_seconds_count"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
def sample(registry, name, operation):
return registry.get_sample_value(name, {"operation": operation})
class Request:
def __init__(self, engine):
self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine))
class Engine:
"""Holds every engine call open until the test releases it."""
def __init__(self):
self.started = asyncio.Event()
self.release = asyncio.Event()
def __getattr__(self, name):
async def call(*args, **kwargs):
self.started.set()
await self.release.wait()
return call
@pytest.fixture
def registry(monkeypatch):
registry = CollectorRegistry()
monkeypatch.setattr(rl_metrics, "_metrics", WeightOperationMetrics(registry))
return registry
@pytest.mark.parametrize("fail", [False, True])
def test_record_releases_in_flight_and_observes_duration(registry, fail):
metrics = rl_metrics.weight_operation_metrics()
with (
pytest.raises(RuntimeError) if fail else nullcontext(),
metrics.record("update"),
):
assert sample(registry, IN_FLIGHT, "update") == 1
if fail:
raise RuntimeError
assert sample(registry, IN_FLIGHT, "update") == 0
assert sample(registry, DURATION, "update") == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"operation,route,body",
[
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
("shutdown", api_router.shutdown_weight_transfer_engine, None),
("start", api_router.start_weight_update, None),
("start_draft", api_router.start_draft_weight_update, None),
("update", api_router.update_weights, {"update_info": {}}),
("finish", api_router.finish_weight_update, None),
],
)
async def test_route_records_its_operation(registry, operation, route, body):
engine = Engine()
task = asyncio.create_task(route(Request(engine), **(body or {})))
await engine.started.wait()
assert sample(registry, IN_FLIGHT, operation) == 1
engine.release.set()
await task
assert sample(registry, IN_FLIGHT, operation) == 0
assert sample(registry, DURATION, operation) == 1
@pytest.mark.asyncio
async def test_cancelled_route_releases_in_flight(registry):
engine = Engine()
task = asyncio.create_task(
api_router.update_weights(Request(engine), update_info={})
)
await engine.started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert sample(registry, IN_FLIGHT, "update") == 0
assert sample(registry, DURATION, "update") == 1
HOLD_OPERATION = """
import sys
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
with weight_operation_metrics().record("update"):
print("entered", flush=True)
sys.stdin.readline()
"""
def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch):
"""Two API-server processes each hold an operation; /metrics sums them."""
monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path))
servers = [
subprocess.Popen(
[sys.executable, "-c", HOLD_OPERATION],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
text=True,
env=os.environ.copy(),
)
for _ in range(2)
]
try:
for server in servers:
assert server.stdout is not None
assert server.stdout.readline().strip() == "entered"
assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2
finally:
for server in servers:
server.communicate("\n", timeout=60)
scraped = get_prometheus_registry()
assert sample(scraped, IN_FLIGHT, "update") == 0
assert sample(scraped, DURATION, "update") == 2
def test_rejected_request_is_not_recorded(registry):
app = FastAPI()
app.include_router(api_router.router)
init_exception_handler(app)
app.state.engine_client = Engine()
app.state.args = SimpleNamespace(log_error_stack=False)
response = TestClient(app).post("/update_weights", json={})
assert response.status_code == 400
assert sample(registry, DURATION, "update") is None
def test_default_recorder_is_created_lazily_on_the_default_registry():
"""The default recorder registers on the process registry only when first used.
Runs in a fresh interpreter: the registry is process-global, so a route test
elsewhere in the session may already have registered these collectors, and
"not registered yet" would then be unobservable.
"""
probe = """
from prometheus_client import REGISTRY
from vllm.entrypoints.rl.online import metrics as rl_metrics
NAMES = [
"vllm:rl_weight_update_operation_duration_seconds",
"vllm:rl_weight_update_operations_in_flight",
]
assert rl_metrics._metrics is None
assert not set(NAMES) & set(REGISTRY._names_to_collectors)
metrics = rl_metrics.weight_operation_metrics()
assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration
assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight
assert rl_metrics.weight_operation_metrics() is metrics
print("lazy registration ok")
"""
result = subprocess.run(
[sys.executable, "-c", probe],
cwd=Path(__file__).parents[3],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert "lazy registration ok" in result.stdout
@@ -1,880 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Round trip parity CI: render -> generate -> derender -> render == render.
Pins the coupled parsing path (`/v1/chat/completions` on a normal GPU
server which parses generated tokens incrementally as they stream out)
against the disaggregated path (`OnlineDerenderer.derender_chat` via
`/v1/chat/completions/derender` which parses the same tokens all at once,
out of process). A standard `vllm serve` GPU server mounts both, so one
real generation lets both parsers run on identical input.
Both paths consume the same generated token IDs (extracted from the coupled
response's `token_ids` via `return_token_ids=True`), so generation
nondeterminism is irrelevant. The only variable under test is whether the
two parsing code paths agree. Parity is asserted unconditionally as only the
stronger per case assertions (e.g. "a tool call was produced") are gated
behind the marker actually having been emitted since a 1.5B model is not
guaranteed to emit `<think>` / `<tool_call>`.
The streaming cases do the same for `stream=true`. Each coupled chunk's
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
through the chunked derender path with `stream_state` threaded across calls
the way a client would, so both parsers see the same chunk boundaries.
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
against `/v1/completions/derender` on the same token IDs, batch and streaming,
plus logprob resolution and stop string handling.
"""
import json
import httpx
import pytest
import pytest_asyncio
import regex as re
from tests.entrypoints.scale_out.derender.utils import (
assemble_stream,
stream_chat_derender,
)
from tests.utils import RemoteOpenAIServer
MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
ARGS = [
"--enable-auto-tool-choice",
"--tool-call-parser",
"hermes",
"--reasoning-parser",
"deepseek_r1",
"--enable-scale-out",
]
TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
@pytest.fixture(scope="module")
def server():
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with httpx.AsyncClient(
base_url=server.url_for(""), timeout=60.0
) as http_client:
yield http_client
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
async def _coupled(client: httpx.AsyncClient, messages: list[dict], **extra) -> dict:
resp = await client.post(
"/v1/chat/completions",
json={
"model": MODEL,
"messages": messages,
"temperature": 0,
"max_tokens": 128,
"return_token_ids": True,
**extra,
},
)
assert resp.status_code == 200, resp.text
return resp.json()
async def _disagg(
client: httpx.AsyncClient,
output_ids: list[int],
prompt_tokens: int,
finish_reason: str,
chat_request: dict,
logprobs: dict | None = None,
) -> dict:
resp = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL,
"generate_response": {
"request_id": "parity",
"choices": [
{
"index": 0,
"token_ids": output_ids,
"finish_reason": finish_reason,
"logprobs": logprobs,
}
],
},
"prompt_tokens": prompt_tokens,
"chat_request": {"return_token_ids": True, **chat_request},
},
)
assert resp.status_code == 200, resp.text
return resp.json()
def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
"""[(name, json normalized args)] so key ordering / whitespace don't
cause false negatives."""
return [
(tc["function"]["name"], json.loads(tc["function"]["arguments"]))
for tc in (response_choice["message"].get("tool_calls") or [])
]
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
return [
_tool_call_id_shape(tc["id"])
for tc in (response_choice["message"].get("tool_calls") or [])
]
def _engine_finish_reason(coupled_finish_reason: str) -> str:
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
gets what the engine reported and has to reapply the rewrite itself."""
if coupled_finish_reason == "tool_calls":
return "stop"
return coupled_finish_reason
def _assert_parity(coupled: dict, disagg: dict) -> None:
"""Both paths saw the same tokens, so they must agree unconditionally."""
c, d = coupled["choices"][0], disagg["choices"][0]
assert d["message"]["content"] == c["message"]["content"]
assert d["message"].get("reasoning") == c["message"].get("reasoning")
assert _tool_sig(d) == _tool_sig(c)
assert _tool_id_shapes(d) == _tool_id_shapes(c)
assert d["finish_reason"] == c["finish_reason"]
assert d["token_ids"] == c["token_ids"]
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
assert disagg["usage"]["completion_tokens"] == len(c["token_ids"])
async def _run_parity_case(
client: httpx.AsyncClient, messages: list[dict], **extra
) -> tuple[dict, dict]:
"""Run the coupled request then feed its generated tokens into the
disaggregated derender endpoint. Returns (coupled, disagg)."""
coupled = await _coupled(client, messages, **extra)
ch = coupled["choices"][0]
chat_request = {"model": MODEL, "messages": messages, **extra}
disagg = await _disagg(
client,
ch["token_ids"],
coupled["usage"]["prompt_tokens"],
_engine_finish_reason(ch["finish_reason"]),
chat_request,
)
return coupled, disagg
# ---------------------------------------------------------------------------
# Parity cases
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_parity_plain(client):
"""Plain detokenization parity. No reasoning/tool markers involved."""
messages = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
coupled, disagg = await _run_parity_case(client, messages)
_assert_parity(coupled, disagg)
@pytest.mark.asyncio
async def test_parity_reasoning(client):
"""Reasoning/content split parity for <think>...</think> outputs."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
coupled, disagg = await _run_parity_case(
client, messages, include_reasoning=True, max_tokens=256
)
_assert_parity(coupled, disagg)
if not coupled["choices"][0]["message"].get("reasoning"):
pytest.skip("Model did not emit a <think> block")
assert disagg["choices"][0]["message"]["reasoning"]
@pytest.mark.asyncio
async def test_parity_tool_call(client):
"""Tool call name+args parity."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, disagg = await _run_parity_case(
client, messages, tools=TOOLS, tool_choice=FORCE_WEATHER_TOOL, max_tokens=1024
)
_assert_parity(coupled, disagg)
if not _tool_sig(coupled["choices"][0]):
pytest.skip("Model did not emit a tool call")
assert _tool_sig(disagg["choices"][0])
@pytest.mark.asyncio
async def test_parity_reasoning_and_tool_call(client):
"""Combined reasoning + tool call parity means the highest drift risk
since it exercises both parser branches on the same output."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, disagg = await _run_parity_case(
client,
messages,
tools=TOOLS,
tool_choice=FORCE_WEATHER_TOOL,
include_reasoning=True,
max_tokens=1024,
)
_assert_parity(coupled, disagg)
c_msg = coupled["choices"][0]["message"]
if not (c_msg.get("reasoning") and _tool_sig(coupled["choices"][0])):
pytest.skip("Model did not emit both a <think> block and a tool call")
d_msg = disagg["choices"][0]["message"]
assert d_msg["reasoning"]
assert _tool_sig(disagg["choices"][0])
def _to_generate_logprobs(openai_logprobs: dict) -> dict:
"""Turn a coupled response's `token_id:N` logprobs into `GenerateLogProbs`.
A real GPU-less worker emits integer ids directly; here the coupled server
with `return_tokens_as_token_ids=True` is the only source of the same ids,
so the placeholders are parsed back out to build the generate-shaped
payload that `/derender` now consumes.
"""
def _id(token: str) -> int:
assert token.startswith("token_id:"), token
return int(token.removeprefix("token_id:"))
return {
"content": [
{
"token_id": _id(entry["token"]),
"logprob": entry["logprob"],
"top_logprobs": [
{"token_id": _id(top["token"]), "logprob": top["logprob"]}
for top in entry["top_logprobs"]
],
}
for entry in openai_logprobs["content"]
]
}
@pytest.mark.asyncio
async def test_parity_logprobs(client):
"""Integer-id logprob decoding parity vs. the coupled server's strings.
A real disaggregated worker only has token IDs, so it emits
`GenerateLogProbs` with integer `token_id`s. `/derender` must decode those
to the same token strings/bytes the coupled server produces directly.
"""
messages = [{"role": "user", "content": "What is 2+2?"}]
extra = {"logprobs": True, "top_logprobs": 3}
# The coupled server with placeholders is used only as a source of the
# token ids a GPU-less worker would have had.
placeholder = await _coupled(
client, messages, return_tokens_as_token_ids=True, **extra
)
ch = placeholder["choices"][0]
chat_request = {"model": MODEL, "messages": messages, **extra}
disagg = await _disagg(
client,
ch["token_ids"],
placeholder["usage"]["prompt_tokens"],
ch["finish_reason"],
chat_request,
logprobs=_to_generate_logprobs(ch["logprobs"]),
)
# The coupled server resolving the same greedy generation
# to real token strings itself.
resolved = await _coupled(client, messages, **extra)
assert resolved["choices"][0]["token_ids"] == ch["token_ids"], (
"greedy (temperature=0) generation was expected to be deterministic "
"across the two coupled calls used to build this test's fixtures"
)
_assert_parity(resolved, disagg)
r_content = resolved["choices"][0]["logprobs"]["content"]
d_content = disagg["choices"][0]["logprobs"]["content"]
assert len(d_content) == len(r_content)
for d_entry, r_entry in zip(d_content, r_content):
assert d_entry["token"] == r_entry["token"]
assert d_entry["bytes"] == r_entry["bytes"]
# ---------------------------------------------------------------------------
# Streaming helpers
# ---------------------------------------------------------------------------
async def _coupled_stream(
client: httpx.AsyncClient,
messages: list[dict],
require_token_ids: bool = True,
**extra,
) -> tuple[list[dict], list[int] | None]:
"""Stream `/v1/chat/completions` and return the choices that carry
`token_ids` along with the prompt token IDs.
The leading role only chunk has no `token_ids` and no counterpart on
`/inference/v1/generate`, so it is dropped. With `require_token_ids`
off every choice is kept and `prompt_token_ids` may be `None`, for
requests where the coupled server hides token ids (hidden reasoning).
"""
choices: list[dict] = []
prompt_token_ids: list[int] | None = None
async with client.stream(
"POST",
"/v1/chat/completions",
json={
"model": MODEL,
"messages": messages,
"temperature": 0,
"max_tokens": 128,
"return_token_ids": True,
"stream": True,
**extra,
},
) as resp:
assert resp.status_code == 200, await resp.aread()
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
data = line[len("data: ") :]
if data == "[DONE]":
break
chunk = json.loads(data)
assert "error" not in chunk, chunk
if chunk.get("prompt_token_ids") is not None:
prompt_token_ids = chunk["prompt_token_ids"]
choices.extend(
ch
for ch in chunk["choices"]
if not require_token_ids or ch.get("token_ids") is not None
)
if require_token_ids:
assert prompt_token_ids is not None
return choices, prompt_token_ids
def _tool_call_id_shape(tool_call_id: str | None) -> str | None:
"""Drop the random suffix so IDs compare by format, e.g. `chatcmpl-tool-`.
Deterministic IDs such as kimi_k2's `functions.<name>:<idx>` are kept
whole."""
if tool_call_id is None:
return None
return re.sub(r"[0-9a-f]{16}$", "", tool_call_id)
async def _run_stream_parity_case(
client: httpx.AsyncClient,
messages: list[dict],
finish_only_tail: bool = False,
**extra,
) -> tuple[dict, list[dict]]:
"""Stream the coupled request, replay its chunks through the streaming
derender endpoint and assert the assembled messages agree. Returns the
assembled coupled message and the coupled choices.
Derender gets the engine's finish reason, since `"tool_calls"` is a chat
level rewrite of `"stop"` that derender has to reapply itself. With
`finish_only_tail` it goes on a trailing chunk with no tokens.
"""
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
finish_reason = coupled_choices[-1]["finish_reason"]
disagg = await stream_chat_derender(
client,
[tid for ch in coupled_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in coupled_choices]
+ ([0] if finish_only_tail else []),
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason=_engine_finish_reason(finish_reason),
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
for tc in message["tool_calls"]:
tc["id"] = _tool_call_id_shape(tc["id"])
assert disagg == coupled
return coupled, coupled_choices
# ---------------------------------------------------------------------------
# Streaming parity cases
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_stream_parity_plain(client):
"""Plain streamed detokenization parity."""
messages = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
await _run_stream_parity_case(client, messages)
@pytest.mark.asyncio
async def test_stream_parity_reasoning(client):
"""Streamed reasoning/content split parity."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
coupled, _ = await _run_stream_parity_case(
client, messages, include_reasoning=True, max_tokens=256
)
if not coupled["reasoning"]:
pytest.skip("Model did not emit a <think> block")
@pytest.mark.asyncio
async def test_stream_parity_tool_call(client):
"""Streamed tool call parity under `tool_choice="auto"`, where a tool
call also rewrites `finish_reason` to `"tool_calls"`."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, _ = await _run_stream_parity_case(
client, messages, tools=TOOLS, tool_choice="auto", max_tokens=512
)
if not coupled["tool_calls"]:
pytest.skip("Model did not emit a tool call")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"chunking", ["engine_steps", "multi_token", "finish_only_tail"]
)
async def test_stream_parity_reasoning_and_tool_call(client, chunking):
"""Streamed reasoning + forced tool call parity across chunk shapes.
`multi_token` batches tokens per chunk with `stream_interval` the way
speculative decoding does, so replay has to reuse the original chunk
boundaries. `finish_only_tail` sends the finish reason on a chunk with no
tokens, which must still flush buffered tool call arguments.
"""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
extra: dict = {
"tools": TOOLS,
"tool_choice": FORCE_WEATHER_TOOL,
"include_reasoning": True,
"max_tokens": 512,
}
if chunking == "multi_token":
extra["stream_interval"] = 4
coupled, coupled_choices = await _run_stream_parity_case(
client,
messages,
finish_only_tail=chunking == "finish_only_tail",
**extra,
)
if chunking == "multi_token":
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
if not (coupled["reasoning"] and coupled["tool_calls"]):
pytest.skip("Model did not emit both a <think> block and a tool call")
# ---------------------------------------------------------------------------
# Streaming logprobs on the parser path
# ---------------------------------------------------------------------------
def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
lp = choice.get("logprobs")
if lp is None or lp.get("content") is None:
return None
return [(e["token"], e.get("bytes")) for e in lp["content"]]
def _sampled_generate_logprobs(choice: dict) -> dict | None:
"""`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
and per-token logprob values, without `top_logprobs`.
The coupled stream's `token_ids` are the integer ids a GPU-less worker
would have sent, so one stream is both the derender input and the string
reference. A second greedy run is not needed, and greedy decoding is not
bit-reproducible across calls on every platform anyway.
"""
lp = choice.get("logprobs")
if lp is None or lp.get("content") is None:
return None
content = lp["content"]
assert len(content) == len(choice["token_ids"]), choice
return {
"content": [
{"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
for tid, entry in zip(choice["token_ids"], content)
]
}
async def _run_stream_logprob_parity_case(
client: httpx.AsyncClient, messages: list[dict], **extra
) -> tuple[dict, list[dict], list[dict]]:
"""Stream with logprobs through the coupled server, replay the same
chunks through the streaming derender endpoint and compare per chunk.
Returns the assembled coupled message, the coupled choices and the
derender choices.
"""
extra = {"logprobs": True, **extra}
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
finish_reason = coupled_choices[-1]["finish_reason"]
derender_choices: list[dict] = []
disagg = await stream_chat_derender(
client,
[tid for ch in coupled_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in coupled_choices],
{"model": MODEL, "messages": messages, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
raw_choices=derender_choices,
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
for tc in message["tool_calls"]:
tc["id"] = _tool_call_id_shape(tc["id"])
assert disagg == coupled
assert len(derender_choices) == len(coupled_choices)
for ref, der in zip(coupled_choices, derender_choices):
assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
return coupled, coupled_choices, derender_choices
@pytest.mark.asyncio
async def test_stream_parity_logprobs_reasoning(client):
"""Per-chunk logprobs on the parser path match the coupled stream across
the reasoning -> content transition."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
client, messages, include_reasoning=True, max_tokens=256
)
if not coupled["reasoning"]:
pytest.skip("Model did not emit a <think> block")
assert any(
ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
)
@pytest.mark.asyncio
async def test_stream_parity_logprobs_tool_call(client):
"""Chunks the parser swallows (tool call markup) still carry their
logprobs, on both sides, so no token's logprobs are lost."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
client,
messages,
tools=TOOLS,
tool_choice=FORCE_WEATHER_TOOL,
include_reasoning=True,
max_tokens=512,
)
if not coupled["tool_calls"]:
pytest.skip("Model did not emit a tool call")
swallowed = [
ch
for ch in derender_choices
if ch.get("logprobs")
and not ch["delta"].get("content")
and not ch["delta"].get("reasoning")
and not ch["delta"].get("tool_calls")
]
assert swallowed, "expected at least one buffered-markup chunk with logprobs"
@pytest.mark.asyncio
async def test_stream_parity_logprobs_hidden_reasoning(client):
"""With `include_reasoning=False` and a parser configured, neither side
returns logprobs on any chunk; derender still emits the content."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
extra = {"logprobs": True, "max_tokens": 256}
# Token ids come from a shown-reasoning run: the coupled server hides
# them along with the logprobs when reasoning is hidden.
shown_choices, prompt_token_ids = await _coupled_stream(
client, messages, include_reasoning=True, **extra
)
hidden_choices, _ = await _coupled_stream(
client, messages, require_token_ids=False, include_reasoning=False, **extra
)
assert all(ch.get("logprobs") is None for ch in hidden_choices)
assert assemble_stream(hidden_choices)["reasoning"] is None
finish_reason = shown_choices[-1]["finish_reason"]
derender_choices: list[dict] = []
disagg = await stream_chat_derender(
client,
[tid for ch in shown_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in shown_choices],
{"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
raw_choices=derender_choices,
)
assert all(ch.get("logprobs") is None for ch in derender_choices)
shown = assemble_stream(shown_choices)
assert disagg["reasoning"] is None
assert disagg["content"] == shown["content"]
# ---------------------------------------------------------------------------
# Inline text level: generate(output_mode="text") == derender(generate(...))
# ---------------------------------------------------------------------------
TEXT_MESSAGES = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
async def _render_token_ids(
client: httpx.AsyncClient, messages: list[dict]
) -> list[int]:
resp = await client.post(
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
)
assert resp.status_code == 200, resp.text
return resp.json()["token_ids"]
def _generate_payload(
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
) -> dict:
return {
"model": MODEL,
"token_ids": token_ids,
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
"output_mode": output_mode,
"stream": stream,
}
async def _generate(
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
) -> dict:
resp = await client.post(
"/inference/v1/generate",
json=_generate_payload(token_ids, output_mode, **sampling),
)
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["output_mode"] == output_mode
return data
async def _generate_text_stream(
client: httpx.AsyncClient, token_ids: list[int], **sampling
) -> list[dict]:
"""Stream generate at the text level and return the choice of every chunk
that has one."""
choices: list[dict] = []
async with client.stream(
"POST",
"/inference/v1/generate",
json=_generate_payload(token_ids, "text", stream=True, **sampling),
) as resp:
assert resp.status_code == 200, await resp.aread()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
assert "error" not in chunk, chunk
assert chunk["output_mode"] == "text"
choices.extend(chunk["choices"])
return choices
async def _derender_completion_text(
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
) -> str:
resp = await client.post(
"/v1/completions/derender",
json={
"model": MODEL,
"generate_responses": [
{
"request_id": "text-parity",
"choices": [
{
"index": 0,
"token_ids": output_ids,
"finish_reason": finish_reason,
}
],
}
],
},
)
assert resp.status_code == 200, resp.text
return resp.json()["choices"][0]["text"]
async def _derender_completion_stream_text(
client: httpx.AsyncClient, choices: list[dict]
) -> str:
"""Replay each generate chunk's token IDs through the streaming
completions derender endpoint with the same chunk boundaries."""
state = None
text = ""
for choice in choices:
resp = await client.post(
"/v1/completions/derender",
json={
"stream": True,
"model": MODEL,
"generate_chunk": {
"request_id": "text-parity-stream",
"choices": [
{
"index": 0,
"token_ids": choice.get("token_ids") or [],
"finish_reason": choice.get("finish_reason"),
}
],
},
"stream_state": state,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
state = data["stream_state"]
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
return text
@pytest.mark.asyncio
async def test_text_parity_batch(client):
"""Inline text matches /v1/completions/derender on the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
inline = (await _generate(client, token_ids, "text"))["choices"][0]
derendered = await _derender_completion_text(
client, inline["token_ids"], inline["finish_reason"]
)
assert inline["text"]
assert inline["text"] == derendered
@pytest.mark.asyncio
async def test_text_parity_stream(client):
"""Streamed inline text matches both the streaming and the non-streaming
derender of the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
choices = await _generate_text_stream(client, token_ids)
inline = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
assert finish_reason is not None
assert inline == await _derender_completion_stream_text(client, choices)
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
@pytest.mark.asyncio
async def test_text_parity_logprobs(client):
"""Inline logprob tokens and bytes match /derender resolving the
token_id:N placeholders of the tokens level."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
sampling = {"logprobs": 3}
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
"choices"
][0]
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
"greedy (temperature=0) generation was expected to be deterministic "
"across the two generate calls"
)
disagg = await _disagg(
client,
tokens_choice["token_ids"],
len(token_ids),
tokens_choice["finish_reason"],
{"model": MODEL, "messages": TEXT_MESSAGES},
logprobs=tokens_choice["logprobs"],
)
inline_content = text_choice["logprobs"]["content"]
derender_content = disagg["choices"][0]["logprobs"]["content"]
assert len(inline_content) == len(derender_content)
for inline_entry, derender_entry in zip(inline_content, derender_content):
assert inline_entry["token"] == derender_entry["token"]
assert inline_entry["bytes"] == derender_entry["bytes"]
assert [
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_text_parity_stop_string(client, stream):
"""Inline text follows the engine's stop handling, like /v1/completions.
token_ids keep every generated token, so /derender brings the matched
stop string back. Its text must start with the inline text and only
differ after it.
"""
stop = "\n"
token_ids = await _render_token_ids(
client,
[{"role": "user", "content": "Write two short lines about the sea."}],
)
resp = await client.post(
"/v1/completions",
json={
"model": MODEL,
"prompt": token_ids,
"temperature": 0,
"max_tokens": 128,
"stop": [stop],
},
)
assert resp.status_code == 200, resp.text
coupled = resp.json()["choices"][0]
if stream:
choices = await _generate_text_stream(client, token_ids, stop=[stop])
inline_text = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
else:
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
inline_text = inline["text"]
output_ids = inline["token_ids"]
finish_reason = inline["finish_reason"]
assert inline_text == coupled["text"]
assert finish_reason == coupled["finish_reason"]
assert stop not in inline_text
derendered = await _derender_completion_text(client, output_ids, finish_reason)
assert derendered.startswith(inline_text)
if coupled["stop_reason"] != stop:
pytest.skip("Model did not emit the stop string")
assert derendered[len(inline_text) :].startswith(stop)
@@ -1,118 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers shared by the streaming chat derender tests."""
from collections.abc import Callable
import httpx
def assemble_stream(choices: list[dict]) -> dict:
"""Fold streamed chat choices into one message.
A tool call index must keep the first ID it was given. On the derender
side that is what pinning the IDs in `stream_state` guarantees.
"""
content = ""
reasoning = ""
tool_calls: list[dict] = []
finish_reason = None
for choice in choices:
delta = choice["delta"]
content += delta.get("content") or ""
reasoning += delta.get("reasoning") or ""
for tc in delta.get("tool_calls") or []:
idx = tc["index"]
while len(tool_calls) <= idx:
tool_calls.append({"id": None, "name": None, "arguments": ""})
if tc.get("id"):
assert tool_calls[idx]["id"] in (None, tc["id"]), (
f"tool call {idx} changed ID mid stream"
)
tool_calls[idx]["id"] = tc["id"]
fn = tc.get("function") or {}
if fn.get("name"):
tool_calls[idx]["name"] = fn["name"]
tool_calls[idx]["arguments"] += fn.get("arguments") or ""
finish_reason = choice.get("finish_reason") or finish_reason
return {
"content": content or None,
"reasoning": reasoning or None,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
async def stream_chat_derender(
client: httpx.AsyncClient,
output_ids: list[int],
chunk_sizes: list[int],
chat_request: dict,
prompt_tokens: int,
prompt_token_ids: list[int],
finish_reason: str = "stop",
on_chunk: Callable[[list[dict]], None] | None = None,
logprobs: list[dict | None] | None = None,
raw_choices: list[dict] | None = None,
) -> dict:
"""Feed `output_ids` through the streaming chat derender endpoint in
the given `chunk_sizes`, threading `stream_state` across calls and
return the assembled message.
`finish_reason` goes on the last chunk, so a trailing size of 0 sends it
on a chunk with no tokens.
If `on_chunk` is given, it is called after every chunk with the
`tool_calls` assembled so far, letting callers assert properties of the
intermediate deltas (e.g. monotonic argument growth) rather than only
the final assembled result.
`logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
and `raw_choices`, if given, collects every derendered choice so callers
can compare per-chunk fields such as `logprobs`.
"""
if logprobs is not None:
assert len(logprobs) == len(chunk_sizes)
state = None
choices: list[dict] = []
pos = 0
for i, size in enumerate(chunk_sizes):
tids = output_ids[pos : pos + size]
pos += size
is_last = i == len(chunk_sizes) - 1
resp = await client.post(
"/v1/chat/completions/derender",
json={
"stream": True,
"model": chat_request["model"],
"generate_chunk": {
"request_id": "stream-test",
"choices": [
{
"index": 0,
"token_ids": tids,
"finish_reason": finish_reason if is_last else None,
"logprobs": logprobs[i] if logprobs is not None else None,
}
],
},
"stream_state": state,
"prompt_tokens": prompt_tokens,
"prompt_token_ids": prompt_token_ids,
"chat_request": chat_request,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
if chat_request.get("return_token_ids") and chat_request.get(
"include_reasoning", True
):
assert data["chunk"]["choices"][0]["token_ids"] == tids
state = data["stream_state"]
choices.extend(data["chunk"]["choices"])
if raw_choices is not None:
raw_choices.extend(data["chunk"]["choices"])
if on_chunk is not None:
on_chunk(assemble_stream(choices)["tool_calls"])
return assemble_stream(choices)
@@ -1,43 +0,0 @@
# Scale-out EC connector e2e tests
End-to-end tests for the disaggregated multimodal (encoder-cache) flow over
the scale-out endpoints, in the bash-script style of
`tests/v1/kv_connector/nixl_integration`.
## Topology
```text
render (GPU-less) encode (EC producer)
/v1/chat/completions/render /inference/v1/generate (kwargs_data)
/v1/chat/completions/derender |
prefill (EC consumer)
/inference/v1/generate (mm_metadata + ec_transfer_params)
```
- The **render** server (`vllm launch render`) preprocesses the chat request
into token ids and multimodal features (`kwargs_data` + `mm_metadata`).
- The **encode** instance (`ec_role: ec_producer`, encode-only) runs the
vision encoder on `kwargs_data` and publishes embeddings through the
`ECExampleConnector` shared storage. Its response carries
`ec_transfer_params`.
- The **prefill** instance (`ec_role: ec_consumer`) receives metadata-only
features (`mm_metadata`, no `kwargs_data`) plus `ec_transfer_params`, loads
the embeddings through the EC connector, and generates output token ids.
- The **derender** endpoint turns the output token ids back into a chat
completion response.
## Running
```bash
bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
```
The script first collects baseline outputs from a single `vllm serve`
instance, then runs the scale-out topology and compares the derendered
outputs for exact equality. It also checks that metadata-only features are
rejected when `ec_transfer_params` is missing.
Defaults: `Qwen/Qwen3-VL-2B-Instruct`, encode on GPU 0, prefill on GPU 1.
Override via environment variables (`MODEL`, `GPU_E`, `GPU_PD`,
`EC_SHARED_STORAGE_PATH`, `PYTHON=.venv/bin/python`, ...); see the header of
the script for the full list.
@@ -1,277 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Scale-out EC connector E2E test
#
# Tests the disaggregated multimodal flow over the scale-out endpoints,
# following the bash-script style of tests/v1/kv_connector/nixl_integration:
#
# 1. Baseline: a single vLLM instance serving /v1/chat/completions
# 2. Scale-out:
# render (GPU-less, `vllm launch render`)
# encode (EC producer, encode-only)
# prefill (EC consumer)
#
# The Python client renders each multimodal request once, sends the full
# kwargs_data to the encode instance, then sends metadata-only features
# (mm_metadata, no kwargs_data) plus ec_transfer_params to the prefill
# instance, and derenders the output tokens. Outputs are compared against
# the baseline for exact equality.
#
# Usage:
# bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
#
# Requires 2 GPUs by default (encode on GPU 0, prefill on GPU 1); the
# baseline reuses GPU 0 after the encode instance is stopped.
set -euo pipefail
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
# Model and engine configuration
MODEL="${MODEL:-Qwen/Qwen3-VL-2B-Instruct}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-8192}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
# GPU configuration. encode + prefill run concurrently on separate GPUs;
# the baseline reuses GPU_E after the scale-out instances are stopped.
GPU_E="${GPU_E:-0}"
GPU_PD="${GPU_PD:-1}"
GPU_SINGLE="${GPU_SINGLE:-$GPU_E}"
# Ports
RENDER_PORT="${RENDER_PORT:-19600}"
ENCODE_PORT="${ENCODE_PORT:-19601}"
PREFILL_PORT="${PREFILL_PORT:-19602}"
BASELINE_PORT="${BASELINE_PORT:-19603}"
# Storage path for the ECExampleConnector encoder cache
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_scale_out_test}"
# Python interpreter (override to .venv/bin/python for local runs)
PYTHON="${PYTHON:-python3}"
# Output files and logs
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_scale_out_ec_baseline.txt}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
BASELINE_LOG="${LOG_PATH}/scale_out_ec_baseline.log"
RENDER_LOG="${LOG_PATH}/scale_out_ec_render.log"
ENCODE_LOG="${LOG_PATH}/scale_out_ec_encode.log"
PREFILL_LOG="${LOG_PATH}/scale_out_ec_prefill.log"
mkdir -p "$LOG_PATH"
# Trap the SIGINT signal (triggered by Ctrl+C)
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
wait_for_server() {
local port=$1
local pid=$2
local name=$3
local deadline=$((SECONDS + TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
if ! kill -0 "$pid" 2>/dev/null; then
echo "$name instance exited before becoming healthy"
return 1
fi
if curl --max-time 2 -fsS "http://localhost:${port}/health" \
> /dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for $name instance on port $port"
return 1
}
dump_log() {
local name=$1
local log=$2
echo "--- $name log (last 200 lines)"
if [[ -f "$log" ]]; then
tail -n 200 "$log" || true
else
echo "Log file not found: $log"
fi
}
dump_scale_out_logs() {
dump_log "render" "$RENDER_LOG"
dump_log "encode" "$ENCODE_LOG"
dump_log "prefill" "$PREFILL_LOG"
}
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -TERM -f "vllm serve" || true
pkill -TERM -f "vllm launch render" || true
sleep 3
pkill -9 -f "vllm serve" || true
pkill -9 -f "vllm launch render" || true
sleep 2
}
# Step 1: baseline single instance
run_baseline() {
echo "================================"
echo "Running BASELINE (single instance)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
echo "Starting baseline instance on GPU $GPU_SINGLE, port $BASELINE_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
--port "$BASELINE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.9 \
--max-num-seqs "$MAX_NUM_SEQS" \
> "$BASELINE_LOG" 2>&1 &
local BASELINE_PID=$!
echo "Waiting for baseline instance to start..."
if ! wait_for_server "$BASELINE_PORT" "$BASELINE_PID" "baseline"; then
dump_log "baseline" "$BASELINE_LOG"
return 1
fi
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
--mode baseline \
--service_url "http://localhost:$BASELINE_PORT" \
--model_name "$MODEL" \
--baseline_file "$BASELINE_FILE"; then
dump_log "baseline" "$BASELINE_LOG"
return 1
fi
echo "Stopping baseline instance..."
kill "$BASELINE_PID" 2>/dev/null || true
sleep 2
cleanup_instances
}
# Step 2: render + encode (EC producer) + prefill (EC consumer)
run_scale_out_ec() {
echo "================================"
echo "Running SCALE-OUT EC (render + encode + prefill)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
mkdir -p "$EC_SHARED_STORAGE_PATH"
declare -a PIDS=()
# GPU-less render/derender server
echo "Starting render server on port $RENDER_PORT"
vllm launch render "$MODEL" \
--port "$RENDER_PORT" \
> "$RENDER_LOG" 2>&1 &
PIDS+=("$!")
# Encode-only EC producer instance. It runs the vision encoder and
# publishes embeddings through the ECExampleConnector shared storage.
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_E" \
vllm serve "$MODEL" \
--enable-scale-out \
--port "$ENCODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.01 \
--no-enable-prefix-caching \
--max-num-batched-tokens 114688 \
--max-num-seqs "$MAX_NUM_SEQS" \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$ENCODE_LOG" 2>&1 &
PIDS+=("$!")
# Prefill/decode EC consumer instance. It receives metadata-only
# features plus ec_transfer_params and loads the embeddings that the
# encode instance published.
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_PD" \
vllm serve "$MODEL" \
--enable-scale-out \
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--max-num-seqs "$MAX_NUM_SEQS" \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_consumer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$PREFILL_LOG" 2>&1 &
PIDS+=("$!")
echo "Waiting for render instance..."
if ! wait_for_server "$RENDER_PORT" "${PIDS[0]}" "render"; then
dump_scale_out_logs
return 1
fi
echo "Waiting for encode instance..."
if ! wait_for_server "$ENCODE_PORT" "${PIDS[1]}" "encode"; then
dump_scale_out_logs
return 1
fi
echo "Waiting for prefill instance..."
if ! wait_for_server "$PREFILL_PORT" "${PIDS[2]}" "prefill"; then
dump_scale_out_logs
return 1
fi
echo "All scale-out EC services are up!"
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
--mode disagg \
--render_url "http://localhost:$RENDER_PORT" \
--encode_url "http://localhost:$ENCODE_PORT" \
--prefill_url "http://localhost:$PREFILL_PORT" \
--model_name "$MODEL" \
--baseline_file "$BASELINE_FILE"; then
dump_scale_out_logs
return 1
fi
echo "Stopping scale-out EC instances..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
sleep 2
cleanup_instances
}
# Main execution
echo "================================"
echo "Scale-out EC E2E Test Suite"
echo "Model: $MODEL"
echo "================================"
run_baseline
run_scale_out_ec
rm -f "$BASELINE_FILE"
echo "================================"
echo "All scale-out EC e2e tests finished!"
echo "================================"
@@ -1,276 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for the scale-out EC connector flow (render -> encode -> prefill).
Mimics the bash-script style of ``tests/v1/kv_connector/nixl_integration``:
the instances are started by ``run_scale_out_ec_e2e_test.sh`` and this file is
the HTTP client that drives the disaggregated request path and compares its
output against a single-instance baseline.
Topology under test (started by the bash script):
render (GPU-less) -- /v1/chat/completions/render, /derender
encode (EC producer) -- /inference/v1/generate
prefill (EC consumer) -- /inference/v1/generate
The client renders each multimodal chat request once, sends the full
``kwargs_data`` to the encode instance, then sends metadata-only features
(``mm_metadata`` without ``kwargs_data``) plus the encode response's
``ec_transfer_params`` to the prefill instance, and finally derenders the
output token ids. The derendered text must match the baseline exactly.
Usage (from the bash script, or manually):
# Baseline mode (saves outputs):
python test_scale_out_ec_e2e.py \
--mode baseline \
--service_url http://localhost:19603 \
--model_name Qwen/Qwen3-VL-2B-Instruct \
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
# Disagg mode (compares outputs):
python test_scale_out_ec_e2e.py \
--mode disagg \
--render_url http://localhost:19600 \
--encode_url http://localhost:19601 \
--prefill_url http://localhost:19602 \
--model_name Qwen/Qwen3-VL-2B-Instruct \
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
"""
import argparse
import json
import sys
import time
import requests
from vllm.assets.image import ImageAsset
from vllm.multimodal.utils import encode_image_url
MAX_OUTPUT_LEN = 128
REQUEST_TIMEOUT = 300
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
image_1_url = encode_image_url(image_1)
image_2_url = encode_image_url(image_2)
# Each prompt is a chat completion request body; the same body is sent to the
# baseline instance, to /render, and to /derender (as chat_request).
SAMPLE_PROMPTS: list[dict] = [
{
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image_1_url}},
{"type": "text", "text": "What's in this image?"},
],
}
],
"description": "single image",
},
{
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image_2_url}},
{"type": "image_url", "image_url": {"url": image_1_url}},
{"type": "text", "text": "Describe these 2 images in detail."},
],
}
],
"description": "two images",
},
{
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"description": "text only",
},
]
def chat_request(prompt: dict) -> dict:
return {
"model": None, # filled by the caller
"messages": prompt["messages"],
"max_tokens": MAX_OUTPUT_LEN,
"temperature": 0.0,
"seed": 42,
}
def post_json(url: str, payload: dict) -> requests.Response:
response = requests.post(url, json=payload, timeout=REQUEST_TIMEOUT)
if response.status_code != 200:
raise RuntimeError(
f"POST {url} failed with {response.status_code}: {response.text}"
)
return response
def run_baseline(args) -> None:
outputs = []
for prompt in SAMPLE_PROMPTS:
request = chat_request(prompt) | {"model": args.model_name}
response = post_json(f"{args.service_url}/v1/chat/completions", request).json()
outputs.append(response["choices"][0]["message"]["content"])
print(f"[baseline] {prompt['description']}: {outputs[-1][:80]!r}")
with open(args.baseline_file, "w") as f:
json.dump(outputs, f)
print(f"Baseline saved to {args.baseline_file}")
def _assert_mm_features(features: dict) -> None:
kwargs_data = features.get("kwargs_data") or {}
mm_metadata = features.get("mm_metadata")
assert any(item is not None for items in kwargs_data.values() for item in items), (
f"render response has no kwargs_data items: {features}"
)
assert mm_metadata, "render response is missing mm_metadata"
for modality, hashes in features["mm_hashes"].items():
assert modality in mm_metadata, f"mm_metadata missing modality {modality}"
assert len(mm_metadata[modality]) == len(hashes)
def run_disagg(args) -> None:
with open(args.baseline_file) as f:
baseline = json.load(f)
failures = []
for prompt, expected in zip(SAMPLE_PROMPTS, baseline, strict=True):
request = chat_request(prompt) | {"model": args.model_name}
got = run_disagg_one(args, request)
if got != expected:
failures.append(
f"{prompt['description']}:\n"
f" baseline: {expected!r}\n disagg: {got!r}"
)
else:
print(f"[disagg] {prompt['description']}: matched baseline")
if failures:
print("Output mismatch:")
for failure in failures:
print(failure)
sys.exit(1)
print("All scale-out EC outputs matched the baseline")
def run_disagg_one(args, request: dict) -> str:
# 1. Render: chat request -> token ids + multimodal features (GPU-less).
rendered = post_json(
f"{args.render_url}/v1/chat/completions/render", request
).json()
token_ids = rendered["token_ids"]
features = rendered.get("features")
request_id = rendered["request_id"]
# 2. Encode: full kwargs_data so the vision encoder can run. The
# response carries ec_transfer_params for the prefill instance.
ec_transfer_params = None
if features is not None:
_assert_mm_features(features)
encode_payload = {
"request_id": request_id,
"token_ids": token_ids,
"features": {
"mm_hashes": features["mm_hashes"],
"mm_placeholders": features["mm_placeholders"],
"kwargs_data": features["kwargs_data"],
},
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
}
encode_response = post_json(
f"{args.encode_url}/inference/v1/generate", encode_payload
).json()
ec_transfer_params = encode_response.get("ec_transfer_params")
assert ec_transfer_params, (
f"encode response has no ec_transfer_params: {encode_response}"
)
# 3. Prefill: metadata-only features. kwargs_data is dropped; the
# embeddings arrive through the EC connector.
if features is not None:
prefill_features = {
"mm_hashes": features["mm_hashes"],
"mm_placeholders": features["mm_placeholders"],
"mm_metadata": features["mm_metadata"],
}
_check_metadata_only_rejected_without_ec(args, token_ids, prefill_features)
else:
prefill_features = None
prefill_payload = {
"request_id": request_id,
"token_ids": token_ids,
"sampling_params": {"max_tokens": MAX_OUTPUT_LEN, "temperature": 0.0},
}
if prefill_features is not None:
prefill_payload["features"] = prefill_features
prefill_payload["ec_transfer_params"] = ec_transfer_params
prefill_response = post_json(
f"{args.prefill_url}/inference/v1/generate", prefill_payload
).json()
output_token_ids = prefill_response["choices"][0]["token_ids"]
assert output_token_ids, f"prefill returned no token ids: {prefill_response}"
# 4. Derender: token ids -> chat completion response (GPU-less).
derendered = post_json(
f"{args.render_url}/v1/chat/completions/derender",
{
"model": args.model_name,
"generate_response": prefill_response,
"prompt_tokens": len(token_ids),
"chat_request": request,
},
).json()
return derendered["choices"][0]["message"]["content"]
def _check_metadata_only_rejected_without_ec(
args, token_ids: list[int], prefill_features: dict
) -> None:
"""mm_metadata without kwargs_data must be rejected without EC params."""
payload = {
"token_ids": token_ids,
"features": prefill_features,
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
}
response = requests.post(
f"{args.prefill_url}/inference/v1/generate",
json=payload,
timeout=REQUEST_TIMEOUT,
)
assert response.status_code in (400, 422), (
"metadata-only features without ec_transfer_params should be rejected, "
f"got {response.status_code}: {response.text}"
)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--mode", choices=["baseline", "disagg"], required=True)
parser.add_argument("--service_url", help="baseline single-instance URL")
parser.add_argument("--render_url", help="GPU-less render server URL")
parser.add_argument("--encode_url", help="EC producer (encode) URL")
parser.add_argument("--prefill_url", help="EC consumer (prefill) URL")
parser.add_argument("--model_name", required=True)
parser.add_argument("--baseline_file", required=True)
args = parser.parse_args()
start = time.time()
if args.mode == "baseline":
assert args.service_url, "--service_url is required in baseline mode"
run_baseline(args)
else:
assert args.render_url, "--render_url is required in disagg mode"
assert args.encode_url, "--encode_url is required in disagg mode"
assert args.prefill_url, "--prefill_url is required in disagg mode"
run_disagg(args)
print(f"Mode {args.mode} finished in {time.time() - start:.1f}s")
if __name__ == "__main__":
main()
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for render endpoints via `vllm launch` (GPU-less serving)."""
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@pytest.fixture(scope="module")
def server():
args: list[str] = []
with RemoteLaunchRenderServer(MODEL_NAME, args, max_wait_seconds=120) as srv:
yield srv
@pytest_asyncio.fixture
async def client(server):
async with httpx.AsyncClient(
base_url=server.url_for(""), timeout=30.0
) as http_client:
yield http_client
# -- Chat Completion Render --
@pytest.mark.asyncio
async def test_chat_render_basic(client):
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [{"role": "user", "content": "Hello, how are you?"}],
},
)
assert response.status_code == 200
data = response.json()
# Response should be a GenerateRequest dict
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
assert all(isinstance(t, int) for t in data["token_ids"])
@pytest.mark.asyncio
async def test_chat_render_multi_turn(client):
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
{"role": "user", "content": "How are you?"},
],
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
# -- Completion Render --
@pytest.mark.asyncio
async def test_completion_render_basic(client):
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": "Once upon a time",
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, list)
assert len(data) > 0
first_prompt = data[0]
assert "token_ids" in first_prompt
assert "sampling_params" in first_prompt
assert "model" in first_prompt
assert "request_id" in first_prompt
assert isinstance(first_prompt["token_ids"], list)
assert len(first_prompt["token_ids"]) > 0
assert first_prompt["request_id"].startswith("cmpl-")
@pytest.mark.asyncio
async def test_completion_render_multiple_prompts(client):
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": ["Hello world", "Goodbye world"],
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, list)
assert len(data) == 2
for prompt in data:
assert "token_ids" in prompt
assert "sampling_params" in prompt
assert "model" in prompt
assert "request_id" in prompt
assert len(prompt["token_ids"]) > 0
assert prompt["request_id"].startswith("cmpl-")
@pytest.mark.asyncio
async def test_completion_render_invalid_model(client):
response = await client.post(
"/v1/completions/render",
json={
"model": "nonexistent-model",
"prompt": "Hello",
},
)
assert response.status_code == 404
assert "error" in response.json()
@pytest.mark.asyncio
async def test_render_is_fast(client):
"""Render should complete quickly since there is no inference."""
import time
start = time.perf_counter()
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": "Tell me a very long story about " * 10,
},
)
elapsed = time.perf_counter() - start
assert response.status_code == 200
assert elapsed < 2.0
# -- Health & Models --
@pytest.mark.asyncio
async def test_health_endpoint(client):
response = await client.get("/health")
assert response.status_code == 200
@pytest.mark.asyncio
async def test_models_endpoint(client):
response = await client.get("/v1/models")
assert response.status_code == 200
data = response.json()
assert "data" in data
model_ids = [m["id"] for m in data["data"]]
assert MODEL_NAME in model_ids
@@ -1,882 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the /render endpoints that expose prompt preprocessing."""
from http import HTTPStatus
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.render.api_router import router
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.renderers.online_renderer import OnlineRenderer
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
def _build_serving_render() -> ServingRender:
serving = ServingRender.__new__(ServingRender)
serving.model_config = SimpleNamespace(
max_model_len=100,
is_encoder_decoder=False,
)
serving.default_sampling_params = {}
serving.override_max_tokens = None
serving.tool_server = MagicMock()
serving.online_renderer = MagicMock()
serving.online_renderer.create_error_response = (
OnlineRenderer.create_error_response.__get__(serving.online_renderer)
)
serving.online_renderer.validate_chat_template = (
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
)
serving.online_renderer.trust_request_chat_template = True
serving.online_renderer.parser = None
serving._check_model = AsyncMock(return_value=None)
return serving
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_completion_rejects_beam_search():
serving = _build_serving_render()
serving.online_renderer.render_completion = AsyncMock(
return_value=[{"prompt_token_ids": [7, 8, 9]}]
)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
use_beam_search=True,
n=2,
length_penalty=0.7,
)
response = await serving.render_completion_request(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert (
response.error.message == "Beam search is not supported by the render endpoint"
)
serving.online_renderer.render_completion.assert_not_awaited()
@pytest.mark.skip_global_cleanup
def test_responses_render_route_is_registered():
assert any(route.path == "/v1/responses/render" for route in router.routes)
@pytest.mark.skip_global_cleanup
def test_responses_render_route_documents_not_implemented():
route = next(
route for route in router.routes if route.path == "/v1/responses/render"
)
assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_returns_generate_request_without_stored_state():
serving = _build_serving_render()
serving.online_renderer.render_responses = AsyncMock(
return_value=MagicMock(
messages=[{"role": "user", "content": "Test prompt"}],
engine_input={"prompt_token_ids": [7, 8, 9]},
)
)
request = ResponsesRequest(
model=MODEL_NAME,
input="Test prompt",
max_output_tokens=12,
stream=True,
cache_salt="request-salt",
priority=3,
kv_transfer_params={"do_remote_prefill": True},
ec_transfer_params={"remote": True},
)
response = await serving.render_responses_request(request)
assert response.token_ids == [7, 8, 9]
assert response.request_id == request.request_id
assert response.sampling_params.max_tokens == 12
assert response.model == MODEL_NAME
assert response.stream is True
assert response.cache_salt == "request-salt"
assert response.priority == 3
assert response.kv_transfer_params == {"do_remote_prefill": True}
assert response.ec_transfer_params == {"remote": True}
assert response.sampling_params.extra_args is None
serving.online_renderer.render_responses.assert_awaited_once_with(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=serving.tool_server,
skip_mm_cache=True,
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_chat_hoists_transfer_params_out_of_extra_args():
serving = _build_serving_render()
serving.online_renderer.render_chat = AsyncMock(
return_value=([], [{"prompt_token_ids": [7, 8, 9]}])
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
kv_transfer_params={"do_remote_prefill": True},
ec_transfer_params={"remote": True},
vllm_xargs={"foo": 1},
)
response = await serving.render_chat_request(request)
assert response.kv_transfer_params == {"do_remote_prefill": True}
assert response.ec_transfer_params == {"remote": True}
assert response.sampling_params.extra_args == {"foo": 1}
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_completion_hoists_transfer_params_for_every_prompt():
serving = _build_serving_render()
serving.online_renderer.render_completion = AsyncMock(
return_value=[
{"prompt_token_ids": [7, 8, 9]},
{"prompt_token_ids": [4, 5]},
]
)
request = CompletionRequest(
model=MODEL_NAME,
prompt=["a", "b"],
kv_transfer_params={"do_remote_prefill": True},
ec_transfer_params={"remote": True},
)
responses = await serving.render_completion_request(request)
assert len(responses) == 2
for response in responses:
assert response.kv_transfer_params == {"do_remote_prefill": True}
assert response.ec_transfer_params == {"remote": True}
assert response.sampling_params.extra_args is None
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_rejects_previous_response_id():
serving = _build_serving_render()
serving.online_renderer.render_responses = AsyncMock()
request = ResponsesRequest(
model=MODEL_NAME,
input="Test prompt",
previous_response_id="resp_previous",
)
response = await serving.render_responses_request(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert response.error.param == "previous_response_id"
serving.online_renderer.render_responses.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_rejects_empty_token_ids():
serving = _build_serving_render()
serving.online_renderer.render_responses = AsyncMock(
return_value=MagicMock(
messages=[],
engine_input={"prompt_token_ids": []},
)
)
response = await serving.render_responses_request(
ResponsesRequest(model=MODEL_NAME, input="Test prompt")
)
assert isinstance(response, ErrorResponse)
assert response.error.message == "No token_ids rendered"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_carries_reasoning_parser_kwargs():
"""/v1/responses gives the engine's reasoning parser the effective template
kwargs, so the rendered request must carry them to generate."""
serving = _build_serving_render()
online = serving.online_renderer
online.parser = MagicMock(reasoning_parser_cls=object)
online.chat_template = None
online.chat_template_content_format = "auto"
online.default_chat_template_kwargs = {"enable_thinking": True}
online.effective_chat_template_kwargs = (
OnlineRenderer.effective_chat_template_kwargs.__get__(online)
)
online.render_responses = AsyncMock(
return_value=MagicMock(messages=[], engine_input={"prompt_token_ids": [7]})
)
request = ResponsesRequest(
model=MODEL_NAME,
input="Test prompt",
chat_template_kwargs={"enable_thinking": False},
)
response = await serving.render_responses_request(request)
assert not isinstance(response, ErrorResponse)
assert response.reasoning_parser_kwargs is not None
kwargs = response.reasoning_parser_kwargs.chat_template_kwargs
assert kwargs["enable_thinking"] is False
@pytest.fixture(scope="module")
def server():
args: list[str] = ["--trust-request-chat-template"]
with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with httpx.AsyncClient(
base_url=server.url_for(""), timeout=30.0
) as http_client:
yield http_client
@pytest.mark.asyncio
async def test_responses_render_basic(client):
response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": "When should a Responses handler return an empty string?",
"max_output_tokens": 7,
},
)
assert response.status_code == 200
data = response.json()
assert data["model"] == MODEL_NAME
assert data["request_id"].startswith("resp_")
assert data["sampling_params"]["max_tokens"] == 7
assert data["token_ids"]
@pytest.mark.asyncio
async def test_responses_render_includes_prior_input_items(client):
current_turn = {
"role": "user",
"content": "Which color should I remember?",
}
current_turn_response = await client.post(
"/v1/responses/render",
json={"model": MODEL_NAME, "input": [current_turn]},
)
multi_turn_response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": [
{"role": "user", "content": "Remember the color cobalt."},
{"role": "assistant", "content": "I will remember cobalt."},
current_turn,
],
},
)
assert current_turn_response.status_code == 200
assert multi_turn_response.status_code == 200
current_turn_token_ids = current_turn_response.json()["token_ids"]
multi_turn_token_ids = multi_turn_response.json()["token_ids"]
assert len(multi_turn_token_ids) > len(current_turn_token_ids)
@pytest.mark.asyncio
async def test_responses_render_rejects_previous_response_id_over_http(client):
response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": "Continue the previous response.",
"previous_response_id": "resp_previous",
},
)
assert response.status_code == 400
error = response.json()["error"]
assert error["type"] == "invalid_request_error"
assert error["param"] == "previous_response_id"
@pytest.mark.asyncio
async def test_completion_render_basic(client):
"""Test basic completion render endpoint."""
# Make request to render endpoint
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": "When should a chat-completions handler return an empty string?",
},
)
assert response.status_code == 200
data = response.json()
# Verify response structure - list of GenerateRequest
assert isinstance(data, list)
assert len(data) > 0
# Verify first prompt is a GenerateRequest
first_prompt = data[0]
assert "token_ids" in first_prompt
assert "sampling_params" in first_prompt
assert "model" in first_prompt
assert "request_id" in first_prompt
assert isinstance(first_prompt["token_ids"], list)
assert len(first_prompt["token_ids"]) > 0
assert first_prompt["model"] == MODEL_NAME
assert first_prompt["request_id"].startswith("cmpl-")
@pytest.mark.asyncio
async def test_chat_completion_render_basic(client):
"""Test basic chat completion render endpoint."""
# Make request to render endpoint
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": (
"Returning an empty string for the prompt may be confusing."
),
}
],
},
)
assert response.status_code == 200
data = response.json()
# Verify response structure - should be a GenerateRequest
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
# Verify token IDs are integers and BOS token is present
token_ids = data["token_ids"]
assert all(isinstance(tid, int) for tid in token_ids)
assert token_ids[0] == 1
@pytest.mark.asyncio
async def test_completion_render_multiple_prompts(client):
"""Test completion render with multiple prompts."""
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": ["Hello world", "Goodbye world"],
},
)
assert response.status_code == 200
data = response.json()
# Should return two GenerateRequest items
assert isinstance(data, list)
assert len(data) == 2
# Verify both prompts have GenerateRequest fields
for prompt in data:
assert "token_ids" in prompt
assert "sampling_params" in prompt
assert "model" in prompt
assert "request_id" in prompt
assert len(prompt["token_ids"]) > 0
assert prompt["request_id"].startswith("cmpl-")
CHAT_BODY = {
"model": MODEL_NAME,
"messages": [{"role": "user", "content": "Hello"}],
}
@pytest.mark.asyncio
async def test_chat_render_honors_body_request_id(client):
response = await client.post(
"/v1/chat/completions/render", json={**CHAT_BODY, "request_id": "abc"}
)
assert response.status_code == 200
assert response.json()["request_id"] == "chatcmpl-abc"
@pytest.mark.asyncio
async def test_chat_render_header_request_id_wins(client):
response = await client.post(
"/v1/chat/completions/render",
json={**CHAT_BODY, "request_id": "abc"},
headers={"X-Request-Id": "hdr"},
)
assert response.status_code == 200
assert response.json()["request_id"] == "chatcmpl-hdr"
@pytest.mark.asyncio
async def test_chat_render_generates_unique_request_ids(client):
ids = set()
for _ in range(2):
response = await client.post("/v1/chat/completions/render", json=CHAT_BODY)
assert response.status_code == 200
request_id = response.json()["request_id"]
assert request_id.startswith("chatcmpl-")
ids.add(request_id)
assert len(ids) == 2
@pytest.mark.asyncio
async def test_completion_render_request_id_single_prompt(client):
response = await client.post(
"/v1/completions/render",
json={"model": MODEL_NAME, "prompt": "Hello world", "request_id": "abc"},
)
assert response.status_code == 200
assert [r["request_id"] for r in response.json()] == ["cmpl-abc"]
@pytest.mark.asyncio
async def test_completion_render_request_id_multiple_prompts(client):
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": ["Hello world", "Goodbye world"],
"request_id": "abc",
},
headers={"X-Request-Id": "hdr"},
)
assert response.status_code == 200
assert [r["request_id"] for r in response.json()] == ["cmpl-hdr-0", "cmpl-hdr-1"]
@pytest.mark.asyncio
async def test_chat_completion_render_multi_turn(client):
"""Test chat completion render with multi-turn conversation."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
{"role": "user", "content": "How are you?"},
],
},
)
assert response.status_code == 200
data = response.json()
# Verify tokenization occurred
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
@pytest.mark.asyncio
async def test_chat_completion_render_with_stream_true(client):
"""Render accepts stream params but still returns JSON (non-streamed)."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"stream": True,
"stream_options": {
"include_usage": True,
"continuous_usage_stats": True,
},
"messages": [
{
"role": "user",
"content": "Stream options should be accepted by /render.",
}
],
},
)
assert response.status_code == 200
assert response.headers.get("content-type", "").startswith("application/json")
data = response.json()
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
# /render should preserve stream fields on the returned token-in request.
assert data.get("stream") is True
assert isinstance(data.get("stream_options"), dict)
assert data["stream_options"].get("include_usage") is True
assert data["stream_options"].get("continuous_usage_stats") is True
@pytest.mark.asyncio
async def test_completion_render_error_invalid_model(client):
"""Test completion render with invalid model returns error."""
response = await client.post(
"/v1/completions/render",
json={
"model": "invalid-model-name",
"prompt": "Hello",
},
)
assert response.status_code == 404
data = response.json()
assert "error" in data
@pytest.mark.asyncio
async def test_chat_completion_render_error_invalid_model(client):
"""Test chat completion render with invalid model returns error."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": "invalid-model-name",
"messages": [{"role": "user", "content": "Hello"}],
},
)
assert response.status_code == 404
data = response.json()
assert "error" in data
@pytest.mark.asyncio
async def test_completion_render_no_generation(client):
"""Verify render endpoint does not generate text."""
# This test verifies that calling render is fast (no generation)
import time
start = time.perf_counter()
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": "Tell me a very long story about " * 10,
},
)
elapsed = time.perf_counter() - start
assert response.status_code == 200
# Render should be fast (< 1 second) since no generation
assert elapsed < 1.0
@pytest.mark.asyncio
async def test_chat_completion_render_with_sampling_params(client):
"""Verify sampling params are correctly returned by /render."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [{"role": "user", "content": "Test sampling params"}],
"temperature": 0.123,
"top_p": 0.456,
"frequency_penalty": 1.1,
},
)
assert response.status_code == 200
data = response.json()
assert "sampling_params" in data
sampling_params = data["sampling_params"]
assert sampling_params.get("temperature") == 0.123
assert sampling_params.get("top_p") == 0.456
assert sampling_params.get("frequency_penalty") == 1.1
# Check that internal fields are not present
assert "_all_stop_token_ids" not in sampling_params
@pytest.mark.asyncio
async def test_completion_render_emits_token_offsets(client):
"""With return_token_offsets, /v1/completions/render returns per-token
(start, end) char offsets aligned with token_ids."""
prompt = "Hello, world."
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": prompt,
"return_token_offsets": True,
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, list)
offsets = data[0]["token_offsets"]
assert offsets is not None
assert len(offsets) == len(data[0]["token_ids"])
for start, end in offsets:
assert isinstance(start, int) and isinstance(end, int)
assert 0 <= start <= end <= len(prompt)
@pytest.mark.asyncio
async def test_completion_render_default_no_token_offsets(client):
"""Without the flag, token_offsets must be null (existing responses
unchanged)."""
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": "Hello, world.",
},
)
assert response.status_code == 200
data = response.json()
assert data[0]["token_offsets"] is None
@pytest.mark.asyncio
async def test_chat_render_emits_token_offsets(client):
"""With return_token_offsets, /v1/chat/completions/render returns
per-token offsets relative to the templated prompt string."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [{"role": "user", "content": "Hello, world."}],
"return_token_offsets": True,
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, dict)
offsets = data["token_offsets"]
assert offsets is not None
assert len(offsets) == len(data["token_ids"])
for start, end in offsets:
assert isinstance(start, int) and isinstance(end, int)
assert 0 <= start <= end
@pytest.mark.asyncio
async def test_chat_render_default_no_token_offsets(client):
"""Without the flag, chat render token_offsets must be null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [{"role": "user", "content": "Hello, world."}],
},
)
assert response.status_code == 200
data = response.json()
assert data["token_offsets"] is None
@pytest.mark.asyncio
async def test_completion_render_truncated_token_offsets(client):
"""Truncation must shorten token_offsets together with token_ids.
An explicit truncation_side turns off tokenizer-level truncation, so the
tokenizer returns offsets for the whole prompt and they are reduced
afterwards -- separately from token_ids. GenerateRequest documents that the
two lists have equal length.
"""
prompt = "The quick brown fox jumps over the lazy dog."
keep = 4
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": prompt,
"return_token_offsets": True,
"truncate_prompt_tokens": keep,
"truncation_side": "left",
},
)
assert response.status_code == 200
data = response.json()
token_ids = data[0]["token_ids"]
offsets = data[0]["token_offsets"]
assert len(token_ids) == keep
assert len(offsets) == len(token_ids)
# Equal length is not enough: truncating from the left keeps the *last*
# tokens, so the surviving offsets must cover the end of the prompt.
assert offsets[-1][1] == len(prompt)
assert offsets[0][0] > 0
@pytest.mark.asyncio
async def test_completion_render_multiple_prompts_token_offsets(client):
"""Each prompt in a batch gets its own offsets aligned with its tokens."""
prompts = ["Hello, world.", "Goodbye, world."]
response = await client.post(
"/v1/completions/render",
json={
"model": MODEL_NAME,
"prompt": prompts,
"return_token_offsets": True,
},
)
assert response.status_code == 200
data = response.json()
assert len(data) == len(prompts)
for item, prompt in zip(data, prompts):
offsets = item["token_offsets"]
assert offsets is not None
assert len(offsets) == len(item["token_ids"])
for start, end in offsets:
assert 0 <= start <= end <= len(prompt)
@pytest.mark.asyncio
async def test_messages_render_basic(client):
"""Test basic Anthropic Messages render endpoint."""
response = await client.post(
"/v1/messages/render",
json={
"model": MODEL_NAME,
"max_tokens": 16,
"messages": [{"role": "user", "content": "Render this Anthropic message."}],
},
)
assert response.status_code == 200
data = response.json()
# Single GenerateRequest, like chat render.
assert isinstance(data, dict)
assert "token_ids" in data
assert "sampling_params" in data
assert "model" in data
assert data["model"] == MODEL_NAME
token_ids = data["token_ids"]
assert isinstance(token_ids, list)
assert len(token_ids) > 0
assert all(isinstance(tid, int) for tid in token_ids)
assert token_ids[0] == 1 # BOS
@pytest.mark.asyncio
async def test_messages_render_system_and_multi_turn(client):
"""System field + multi-turn messages render to a single prompt."""
response = await client.post(
"/v1/messages/render",
json={
"model": MODEL_NAME,
"max_tokens": 16,
"system": "You are a helpful assistant.",
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi! How can I help?"},
{"role": "user", "content": "What is 2 + 2?"},
],
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, dict)
assert len(data["token_ids"]) > 0
assert data["token_ids"][0] == 1 # BOS
@pytest.mark.asyncio
async def test_messages_render_merges_inline_system(client):
"""Inline system messages merge into the leading system block.
The model's chat template rejects non-leading system messages, so the
/v1/messages server path detects merge_inline_system=True and render must
produce the same tokens as the manually pre-merged request.
"""
inline = await client.post(
"/v1/messages/render",
json={
"model": MODEL_NAME,
"max_tokens": 16,
"system": "You are a helpful assistant.",
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi! How can I help?"},
{"role": "system", "content": "Be brief."},
{"role": "user", "content": "What is 2 + 2?"},
],
},
)
assert inline.status_code == 200
merged = await client.post(
"/v1/messages/render",
json={
"model": MODEL_NAME,
"max_tokens": 16,
"system": "You are a helpful assistant.Be brief.",
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi! How can I help?"},
{"role": "user", "content": "What is 2 + 2?"},
],
},
)
assert merged.status_code == 200
assert inline.json()["token_ids"] == merged.json()["token_ids"]
@pytest.mark.asyncio
async def test_messages_render_error_invalid_model(client):
"""Messages render with an invalid model returns an error."""
response = await client.post(
"/v1/messages/render",
json={
"model": "invalid-model-name",
"max_tokens": 16,
"messages": [{"role": "user", "content": "Hello"}],
},
)
assert response.status_code == 404
data = response.json()
assert "error" in data
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Multimodal tests for the /render endpoints that expose prompt preprocessing."""
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.utils import encode_image_url
VISION_MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
@pytest.fixture(scope="module")
def vision_server():
"""Vision-capable server used for multimodal /render tests."""
args = [
"--enforce-eager",
"--max-model-len",
"100",
"--max-num-seqs",
"1",
"--limit-mm-per-prompt.image",
"1",
"--limit-mm-per-prompt.video",
"0",
"--enable-scale-out",
]
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def vision_client(vision_server):
async with httpx.AsyncClient(
base_url=vision_server.url_for(""), timeout=60.0
) as http_client:
yield http_client
@pytest.mark.asyncio
async def test_chat_completion_render_with_base64_image_url(
vision_client,
local_asset_server,
):
"""Render a multimodal chat request and verify tokens are returned."""
image = local_asset_server.get_image_asset("RGBA_comp.png")
data_url = encode_image_url(image, format="PNG")
assert data_url.startswith("data:image/")
assert ";base64," in data_url
response = await vision_client.post(
"/v1/chat/completions/render",
json={
"model": VISION_MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What's in this image?"},
],
}
],
},
)
assert response.status_code == 200
data = response.json()
assert isinstance(data, dict)
assert "token_ids" in data
assert isinstance(data["token_ids"], list)
assert len(data["token_ids"]) > 0
# Verify multimodal features are populated
assert "features" in data
features = data["features"]
assert features is not None
# mm_hashes: should have an "image" key with a list of hash strings
assert "mm_hashes" in features
assert "image" in features["mm_hashes"]
image_hashes = features["mm_hashes"]["image"]
assert isinstance(image_hashes, list)
assert len(image_hashes) > 0
assert all(isinstance(h, str) for h in image_hashes)
# mm_placeholders: should have an "image" key with offset/length dicts
assert "mm_placeholders" in features
assert "image" in features["mm_placeholders"]
image_placeholders = features["mm_placeholders"]["image"]
assert isinstance(image_placeholders, list)
assert len(image_placeholders) > 0
for p in image_placeholders:
assert "offset" in p
assert "length" in p
assert isinstance(p["offset"], int)
assert isinstance(p["length"], int)
assert p["length"] > 0
@pytest.mark.asyncio
async def test_tokenize_matches_render_for_multimodal_input(
vision_client,
local_asset_server,
):
"""`/tokenize` should match `/v1/chat/completions/render` token output."""
image = local_asset_server.get_image_asset("RGBA_comp.png")
data_url = encode_image_url(image, format="PNG")
messages = [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What's in this image?"},
],
}
]
render_response = await vision_client.post(
"/v1/chat/completions/render",
json={
"model": VISION_MODEL_NAME,
"messages": messages,
},
)
assert render_response.status_code == 200
render_data = render_response.json()
tokenize_response = await vision_client.post(
"/tokenize",
json={
"model": VISION_MODEL_NAME,
"messages": messages,
},
)
assert tokenize_response.status_code == 200
tokenize_data = tokenize_response.json()
assert tokenize_data["tokens"] == render_data["token_ids"]
assert tokenize_data["count"] == len(render_data["token_ids"])
+3 -18
View File
@@ -17,18 +17,10 @@ from vllm.entrypoints.serve.middleware.authenticate import (
)
from vllm.tasks import SupportedTask
RENDER_PATHS = {
"/v1/chat/completions/render",
"/v1/chat/completions/derender",
"/v1/completions/render",
"/v1/completions/derender",
"/v1/messages/render",
"/v1/responses/render",
}
GENERATE_PATH = "/inference/v1/generate"
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
def registered_paths(
@@ -100,13 +92,6 @@ def test_scale_out_routes_can_be_enabled_for_generate_server():
assert paths >= SCALE_OUT_PATHS
def test_render_routes_remain_enabled_for_render_server():
paths = registered_paths(("render",))
assert paths >= RENDER_PATHS
assert GENERATE_PATH not in paths
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
@@ -124,8 +109,8 @@ def test_abort_route_without_tokens_only_requires_api_key():
def test_unauthenticated_abort_route_is_not_registered_twice():
"""The RLHF dev router registers /abort_requests before the scale-out
routers, so attaching the scale-out one too would add a shadowed duplicate."""
"""A pre-existing /abort_requests route must not be shadowed by a second
registration from the scale-out router."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@@ -8,7 +8,6 @@ import pytest
import torch
from pydantic import ValidationError
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
@@ -195,10 +194,6 @@ _MESSAGES = [{"role": "user", "content": "hi"}]
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
(CompletionRequest, {"model": "m", "prompt": "hi"}),
(ResponsesRequest, {"model": "m", "input": "hi"}),
(
AnthropicMessagesRequest,
{"model": "m", "max_tokens": 8, "messages": _MESSAGES},
),
],
)
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
@@ -120,7 +120,6 @@ class TestAffectedModulesUseSanitize:
@pytest.mark.parametrize(
"module",
[
"vllm.entrypoints.anthropic.serving",
"vllm.entrypoints.speech_to_text.realtime.connection",
],
)
@@ -32,7 +32,6 @@ def _build_args() -> Namespace:
"""Minimal args for ``build_app``; avoids ``make_arg_parser`` device probing."""
return Namespace(
disable_fastapi_docs=True,
enable_offline_docs=False,
root_path=None,
allowed_origins=["*"],
allow_credentials=False,
@@ -1,60 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared fixtures and utilities for SageMaker tests."""
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# Model name constants used across tests
MODEL_NAME_SMOLLM = "HuggingFaceTB/SmolLM2-135M-Instruct"
LORA_ADAPTER_NAME_SMOLLM = "jekunz/smollm-135m-lora-fineweb-faroese"
# SageMaker header constants
HEADER_SAGEMAKER_CLOSED_SESSION_ID = "X-Amzn-SageMaker-Closed-Session-Id"
HEADER_SAGEMAKER_SESSION_ID = "X-Amzn-SageMaker-Session-Id"
HEADER_SAGEMAKER_NEW_SESSION_ID = "X-Amzn-SageMaker-New-Session-Id"
@pytest.fixture(scope="session")
def smollm2_lora_files():
"""Download LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id=LORA_ADAPTER_NAME_SMOLLM)
@pytest.fixture(scope="module")
def basic_server_with_lora(smollm2_lora_files):
"""Basic server fixture with standard configuration."""
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--max-lora-rank",
"256",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"64",
]
envs = {
"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True",
"SAGEMAKER_ENABLE_STATEFUL_SESSIONS": "True",
}
with RemoteOpenAIServer(MODEL_NAME_SMOLLM, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def async_client(basic_server_with_lora: RemoteOpenAIServer):
"""Async OpenAI client fixture for use with basic_server."""
async with basic_server_with_lora.get_async_client() as async_client:
yield async_client
@@ -1,629 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration tests for handler override functionality.
Tests real customer usage scenarios:
- Using @custom_ping_handler and @custom_invocation_handler decorators
to override handlers
- Setting environment variables for handler specifications
- Writing customer scripts with custom_sagemaker_ping_handler() and
custom_sagemaker_invocation_handler() functions
- Priority: env vars > decorators > customer script files > framework
defaults
The handler-override scenarios exercise the real vLLM SageMaker router and
bootstrap path via an in-process FastAPI ``TestClient`` instead of launching a
model server. These scenarios fully replace the ``/ping`` and ``/invocations``
endpoints with customer handlers, so no inference engine is required to
validate override behavior. Avoiding the model server also keeps the tests
fast and deterministic rather than depending on the FastAPI version resolved
into the test environment at runtime.
"""
import os
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from tests.utils import RemoteOpenAIServer
from .conftest import (
MODEL_NAME_SMOLLM,
)
def _build_sagemaker_test_client() -> TestClient:
"""Build a TestClient over the real SageMaker router and bootstrap path.
``attach_router`` is called with empty supported tasks because the override
tests replace the endpoints with customer handlers, so no framework
invocation handler (and therefore no engine) is exercised.
"""
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router,
sagemaker_standards_bootstrap,
)
app = FastAPI()
attach_router(app, ())
return TestClient(sagemaker_standards_bootstrap(app))
class TestHandlerOverrideIntegration:
"""Integration tests simulating real customer usage scenarios.
Each test simulates a fresh server startup where customers:
- Use @custom_ping_handler and @custom_invocation_handler decorators
- Set environment variables (CUSTOM_FASTAPI_PING_HANDLER, etc.)
- Write customer scripts with custom_sagemaker_ping_handler() and
custom_sagemaker_invocation_handler() functions
"""
def setup_method(self):
"""Setup for each test - simulate fresh server startup."""
self._clear_caches()
self._clear_env_vars()
def teardown_method(self):
"""Cleanup after each test."""
self._clear_env_vars()
def _clear_caches(self):
"""Clear handler registry and function loader cache."""
try:
from model_hosting_container_standards.common.handler import (
handler_registry,
)
from model_hosting_container_standards.sagemaker.sagemaker_loader import (
SageMakerFunctionLoader,
)
handler_registry.clear()
SageMakerFunctionLoader._default_function_loader = None
except ImportError:
pytest.skip("model-hosting-container-standards not available")
def _clear_env_vars(self):
"""Clear SageMaker environment variables."""
try:
from model_hosting_container_standards.common.fastapi.config import (
FastAPIEnvVars,
)
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
# Clear SageMaker env vars
for var in [
SageMakerEnvVars.SAGEMAKER_MODEL_PATH,
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME,
]:
os.environ.pop(var, None)
# Clear FastAPI env vars
for var in [
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
]:
os.environ.pop(var, None)
except ImportError:
pass
def test_customer_script_functions_auto_loaded(self, monkeypatch, tmp_path):
"""Test customer scenario: script functions automatically override
framework defaults."""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a script file with ping() and invoke() functions
script_path = tmp_path / "model.py"
script_path.write_text(
"""
from fastapi import Request
async def custom_sagemaker_ping_handler():
return {
"status": "healthy",
"source": "customer_override",
"message": "Custom ping from customer script"
}
async def custom_sagemaker_invocation_handler(request: Request):
return {
"predictions": ["Custom response from customer script"],
"source": "customer_override"
}
"""
)
# Customer sets SageMaker environment variables to point to their script
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
with _build_sagemaker_test_client() as client:
# Customer tests their server and sees their overrides work
# automatically
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# Customer sees their functions are used
assert ping_data["source"] == "customer_override"
assert ping_data["message"] == "Custom ping from customer script"
assert invoke_data["source"] == "customer_override"
assert invoke_data["predictions"] == [
"Custom response from customer script"
]
def test_customer_decorator_usage(self, monkeypatch, tmp_path):
"""Test customer scenario: using @custom_ping_handler and
@custom_invocation_handler decorators."""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a script file with decorators
script_path = tmp_path / "model.py"
script_path.write_text(
"""
import model_hosting_container_standards.sagemaker as sagemaker_standards
from fastapi import Request
@sagemaker_standards.custom_ping_handler
async def my_ping():
return {
"type": "ping",
"source": "customer_decorator"
}
@sagemaker_standards.custom_invocation_handler
async def my_invoke(request: Request):
return {
"type": "invoke",
"source": "customer_decorator"
}
"""
)
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
with _build_sagemaker_test_client() as client:
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# Customer sees their handlers are used by the server
assert ping_data["source"] == "customer_decorator"
assert invoke_data["source"] == "customer_decorator"
def test_handler_priority_order(self, monkeypatch, tmp_path):
"""Test priority: @custom_ping_handler/@custom_invocation_handler
decorators vs script functions."""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a script with both decorator and regular functions
script_path = tmp_path / "model.py"
script_path.write_text(
"""
import model_hosting_container_standards.sagemaker as sagemaker_standards
from fastapi import Request
# Customer uses @custom_ping_handler decorator (higher priority than script functions)
@sagemaker_standards.custom_ping_handler
async def decorated_ping():
return {
"status": "healthy",
"source": "ping_decorator_in_script",
"priority": "decorator"
}
# Customer also has a regular function (lower priority than
# @custom_ping_handler decorator)
async def custom_sagemaker_ping_handler():
return {
"status": "healthy",
"source": "script_function",
"priority": "function"
}
# Customer has a regular invoke function
async def custom_sagemaker_invocation_handler(request: Request):
return {
"predictions": ["Script function response"],
"source": "script_invoke_function",
"priority": "function"
}
"""
)
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
with _build_sagemaker_test_client() as client:
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# @custom_ping_handler decorator has higher priority than
# script function
assert ping_data["source"] == "ping_decorator_in_script"
assert ping_data["priority"] == "decorator"
# Script function is used for invoke
assert invoke_data["source"] == "script_invoke_function"
assert invoke_data["priority"] == "function"
def test_environment_variable_script_loading(self, monkeypatch, tmp_path):
"""Test that environment variables correctly specify script location
and loading."""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a script in a specific directory
script_path = tmp_path / "model.py"
script_path.write_text(
"""
from fastapi import Request
async def custom_sagemaker_ping_handler():
return {
"status": "healthy",
"source": "env_loaded_script",
"method": "environment_variable_loading"
}
async def custom_sagemaker_invocation_handler(request: Request):
return {
"predictions": ["Loaded via environment variables"],
"source": "env_loaded_script",
"method": "environment_variable_loading"
}
"""
)
# Test environment variable script loading
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
with _build_sagemaker_test_client() as client:
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# Verify that the script was loaded via environment variables
assert ping_data["source"] == "env_loaded_script"
assert ping_data["method"] == "environment_variable_loading"
assert invoke_data["source"] == "env_loaded_script"
assert invoke_data["method"] == "environment_variable_loading"
@pytest.mark.asyncio
async def test_framework_default_handlers(self):
"""Test that framework default handlers work when no customer
overrides exist.
This scenario exercises the real inference path (default
``/invocations``), so it keeps using a live model server rather than
the in-process TestClient.
"""
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--enforce-eager",
"--max-num-seqs",
"32",
]
# Explicitly pass empty env_dict to ensure no SageMaker env vars are set
# This prevents pollution from previous tests
try:
from model_hosting_container_standards.common.fastapi.config import (
FastAPIEnvVars,
)
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
env_dict = {
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: "",
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: "",
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER: "",
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER: "",
}
except ImportError:
env_dict = {}
with RemoteOpenAIServer(MODEL_NAME_SMOLLM, args, env_dict=env_dict) as server:
# Test that default ping works
ping_response = requests.get(server.url_for("ping"))
assert ping_response.status_code == 200
# Test that default invocations work
invoke_response = requests.post(
server.url_for("invocations"),
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
def test_handler_env_var_override(self, monkeypatch, tmp_path):
"""Test CUSTOM_FASTAPI_PING_HANDLER and CUSTOM_FASTAPI_INVOCATION_HANDLER
environment variable overrides."""
try:
from model_hosting_container_standards.common.fastapi.config import (
FastAPIEnvVars,
)
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Create a script with both env var handlers and script functions
script_path = tmp_path / "model.py"
script_path.write_text(
"""
from fastapi import Request, Response
import json
async def env_var_ping_handler(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"status": "healthy",
"source": "env_var_ping",
"method": "environment_variable"
}),
media_type="application/json"
)
async def env_var_invoke_handler(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"predictions": ["Environment variable response"],
"source": "env_var_invoke",
"method": "environment_variable"
}),
media_type="application/json"
)
async def custom_sagemaker_ping_handler():
return {
"status": "healthy",
"source": "script_ping",
"method": "script_function"
}
async def custom_sagemaker_invocation_handler(request: Request):
return {
"predictions": ["Script function response"],
"source": "script_invoke",
"method": "script_function"
}
"""
)
# Set environment variables to override both handlers
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
monkeypatch.setenv(
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
f"{script_path.name}:env_var_ping_handler",
)
monkeypatch.setenv(
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
f"{script_path.name}:env_var_invoke_handler",
)
with _build_sagemaker_test_client() as client:
# Test ping handler override
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
# Environment variable should override script function
assert ping_data["method"] == "environment_variable"
assert ping_data["source"] == "env_var_ping"
# Test invocation handler override
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# Environment variable should override script function
assert invoke_data["method"] == "environment_variable"
assert invoke_data["source"] == "env_var_invoke"
def test_env_var_priority_over_decorator_and_script(self, monkeypatch, tmp_path):
"""Test that environment variables have highest priority over decorators
and script functions for both ping and invocation handlers."""
try:
from model_hosting_container_standards.common.fastapi.config import (
FastAPIEnvVars,
)
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Create a script with all three handler types for both ping and invocation
script_path = tmp_path / "model.py"
script_path.write_text(
"""
import model_hosting_container_standards.sagemaker as sagemaker_standards
from fastapi import Request, Response
import json
# Environment variable handlers (highest priority)
async def env_priority_ping(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"status": "healthy",
"source": "env_var",
"priority": "environment_variable"
}),
media_type="application/json"
)
async def env_priority_invoke(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"predictions": ["Environment variable response"],
"source": "env_var",
"priority": "environment_variable"
}),
media_type="application/json"
)
# Decorator handlers (medium priority)
@sagemaker_standards.custom_ping_handler
async def decorator_ping(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"status": "healthy",
"source": "decorator",
"priority": "decorator"
}),
media_type="application/json"
)
@sagemaker_standards.custom_invocation_handler
async def decorator_invoke(raw_request: Request) -> Response:
return Response(
content=json.dumps({
"predictions": ["Decorator response"],
"source": "decorator",
"priority": "decorator"
}),
media_type="application/json"
)
# Script functions (lowest priority)
async def custom_sagemaker_ping_handler():
return {
"status": "healthy",
"source": "script",
"priority": "script_function"
}
async def custom_sagemaker_invocation_handler(request: Request):
return {
"predictions": ["Script function response"],
"source": "script",
"priority": "script_function"
}
"""
)
# Set environment variables to specify highest priority handlers
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
monkeypatch.setenv(
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
f"{script_path.name}:env_priority_ping",
)
monkeypatch.setenv(
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
f"{script_path.name}:env_priority_invoke",
)
with _build_sagemaker_test_client() as client:
# Test ping handler priority
ping_response = client.get("/ping")
assert ping_response.status_code == 200
ping_data = ping_response.json()
# Environment variable has highest priority and should be used
assert ping_data["priority"] == "environment_variable"
assert ping_data["source"] == "env_var"
# Test invocation handler priority
invoke_response = client.post(
"/invocations",
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
},
)
assert invoke_response.status_code == 200
invoke_data = invoke_response.json()
# Environment variable has highest priority and should be used
assert invoke_data["priority"] == "environment_variable"
assert invoke_data["source"] == "env_var"
@@ -1,172 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import openai # use the official async_client for correctness check
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from .conftest import MODEL_NAME_SMOLLM
@pytest.mark.asyncio
async def test_sagemaker_load_adapter_happy_path(
async_client: openai.AsyncOpenAI,
basic_server_with_lora: RemoteOpenAIServer,
smollm2_lora_files,
):
# The SageMaker standards library creates a POST /adapters endpoint
# that maps to the load_lora_adapter handler with request shape:
# {"lora_name": "body.name", "lora_path": "body.src"}
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": "smollm2-lora-sagemaker", "src": smollm2_lora_files},
)
load_response.raise_for_status()
models = await async_client.models.list()
models = models.data
dynamic_lora_model = models[-1]
assert dynamic_lora_model.root == smollm2_lora_files
assert dynamic_lora_model.parent == MODEL_NAME_SMOLLM
assert dynamic_lora_model.id == "smollm2-lora-sagemaker"
@pytest.mark.asyncio
async def test_sagemaker_unload_adapter_happy_path(
async_client: openai.AsyncOpenAI,
basic_server_with_lora: RemoteOpenAIServer,
smollm2_lora_files,
):
# First, load an adapter
adapter_name = "smollm2-lora-sagemaker-unload"
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": adapter_name, "src": smollm2_lora_files},
)
load_response.raise_for_status()
# Verify it's in the models list
models = await async_client.models.list()
adapter_ids = [model.id for model in models.data]
assert adapter_name in adapter_ids
# Now unload it using DELETE /adapters/{adapter_name}
# The SageMaker standards maps this to unload_lora_adapter with:
# {"lora_name": "path_params.adapter_name"}
unload_response = requests.delete(
basic_server_with_lora.url_for("adapters", adapter_name),
)
unload_response.raise_for_status()
# Verify it's no longer in the models list
models = await async_client.models.list()
adapter_ids = [model.id for model in models.data]
assert adapter_name not in adapter_ids
@pytest.mark.asyncio
async def test_sagemaker_load_adapter_not_found(
basic_server_with_lora: RemoteOpenAIServer,
):
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": "nonexistent-adapter", "src": "/path/does/not/exist"},
)
assert load_response.status_code == 404
@pytest.mark.asyncio
async def test_sagemaker_load_adapter_invalid_files(
basic_server_with_lora: RemoteOpenAIServer,
tmp_path,
):
invalid_files = tmp_path / "invalid_adapter"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("not valid json")
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": "invalid-adapter", "src": str(invalid_files)},
)
assert load_response.status_code == 500
@pytest.mark.asyncio
async def test_sagemaker_unload_nonexistent_adapter(
basic_server_with_lora: RemoteOpenAIServer,
):
# Attempt to unload an adapter that doesn't exist
unload_response = requests.delete(
basic_server_with_lora.url_for("adapters", "nonexistent-adapter-name"),
)
assert unload_response.status_code in (400, 404)
@pytest.mark.asyncio
async def test_sagemaker_invocations_with_adapter(
basic_server_with_lora: RemoteOpenAIServer,
smollm2_lora_files,
):
# First, load an adapter via SageMaker endpoint
adapter_name = "smollm2-lora-invoke-test"
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": adapter_name, "src": smollm2_lora_files},
)
load_response.raise_for_status()
# Now test the /invocations endpoint with the adapter
invocation_response = requests.post(
basic_server_with_lora.url_for("invocations"),
headers={
"X-Amzn-SageMaker-Adapter-Identifier": adapter_name,
},
json={
"prompt": "Hello, how are you?",
"max_tokens": 10,
},
)
invocation_response.raise_for_status()
invocation_output = invocation_response.json()
# Verify we got a valid completion response
assert "choices" in invocation_output
assert len(invocation_output["choices"]) > 0
assert "text" in invocation_output["choices"][0]
@pytest.mark.asyncio
async def test_sagemaker_multiple_adapters_load_unload(
async_client: openai.AsyncOpenAI,
basic_server_with_lora: RemoteOpenAIServer,
smollm2_lora_files,
):
adapter_names = [f"sagemaker-adapter-{i}" for i in range(5)]
# Load all adapters
for adapter_name in adapter_names:
load_response = requests.post(
basic_server_with_lora.url_for("adapters"),
json={"name": adapter_name, "src": smollm2_lora_files},
)
load_response.raise_for_status()
# Verify all are in the models list
models = await async_client.models.list()
adapter_ids = [model.id for model in models.data]
for adapter_name in adapter_names:
assert adapter_name in adapter_ids
# Unload all adapters
for adapter_name in adapter_names:
unload_response = requests.delete(
basic_server_with_lora.url_for("adapters", adapter_name),
)
unload_response.raise_for_status()
# Verify all are removed from models list
models = await async_client.models.list()
adapter_ids = [model.id for model in models.data]
for adapter_name in adapter_names:
assert adapter_name not in adapter_ids
@@ -1,347 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration test for middleware loader functionality.
Tests that customer middlewares get called correctly with a vLLM server.
"""
import os
import tempfile
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from .conftest import (
MODEL_NAME_SMOLLM,
)
class TestMiddlewareIntegration:
"""Integration test for middleware with vLLM server."""
def setup_method(self):
"""Setup for each test - simulate fresh server startup."""
self._clear_caches()
def _clear_caches(self):
"""Clear middleware registry and function loader cache."""
try:
from model_hosting_container_standards.common.fastapi.middleware import (
middleware_registry,
)
from model_hosting_container_standards.common.fastapi.middleware.source.decorator_loader import ( # noqa: E501
decorator_loader,
)
from model_hosting_container_standards.sagemaker.sagemaker_loader import (
SageMakerFunctionLoader,
)
middleware_registry.clear_middlewares()
decorator_loader.clear()
SageMakerFunctionLoader._default_function_loader = None
except ImportError:
pytest.skip("model-hosting-container-standards not available")
@pytest.mark.asyncio
async def test_customer_middleware_with_vllm_server(self):
"""Test that customer middlewares work with actual vLLM server.
Tests decorator-based middlewares (@custom_middleware, @input_formatter,
@output_formatter)
on multiple endpoints (chat/completions, invocations).
"""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a middleware script with multiple decorators
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
f.write(
"""
from model_hosting_container_standards.common.fastapi.middleware import (
custom_middleware, input_formatter, output_formatter
)
# Global flag to track if input formatter was called
_input_formatter_called = False
@input_formatter
async def customer_input_formatter(request):
# Process input - mark that input formatter was called
global _input_formatter_called
_input_formatter_called = True
return request
@custom_middleware("throttle")
async def customer_throttle_middleware(request, call_next):
response = await call_next(request)
response.headers["X-Customer-Throttle"] = "applied"
order = response.headers.get("X-Middleware-Order", "")
response.headers["X-Middleware-Order"] = order + "throttle,"
return response
@output_formatter
async def customer_output_formatter(response):
global _input_formatter_called
response.headers["X-Customer-Processed"] = "true"
# Since input_formatter and output_formatter are combined into
# pre_post_process middleware,
# if output_formatter is called, input_formatter should have been called too
if _input_formatter_called:
response.headers["X-Input-Formatter-Called"] = "true"
order = response.headers.get("X-Middleware-Order", "")
response.headers["X-Middleware-Order"] = order + "output_formatter,"
return response
"""
)
script_path = f.name
try:
script_dir = os.path.dirname(script_path)
script_name = os.path.basename(script_path)
# Set environment variables to point to customer script
env_vars = {
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
}
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--enforce-eager",
"--max-num-seqs",
"32",
]
with RemoteOpenAIServer(
MODEL_NAME_SMOLLM, args, env_dict=env_vars
) as server:
# Test 1: Middlewares applied to chat/completions endpoint
chat_response = requests.post(
server.url_for("v1/chat/completions"),
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
"temperature": 0.0,
},
)
assert chat_response.status_code == 200
# Verify all middlewares were executed
assert "X-Customer-Throttle" in chat_response.headers
assert chat_response.headers["X-Customer-Throttle"] == "applied"
assert "X-Customer-Processed" in chat_response.headers
assert chat_response.headers["X-Customer-Processed"] == "true"
# Verify input formatter was called
assert "X-Input-Formatter-Called" in chat_response.headers
assert chat_response.headers["X-Input-Formatter-Called"] == "true"
# Verify middleware execution order
execution_order = chat_response.headers.get(
"X-Middleware-Order", ""
).rstrip(",")
order_parts = execution_order.split(",") if execution_order else []
assert "throttle" in order_parts
assert "output_formatter" in order_parts
# Test 2: Middlewares applied to invocations endpoint
invocations_response = requests.post(
server.url_for("invocations"),
json={
"model": MODEL_NAME_SMOLLM,
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 5,
"temperature": 0.0,
},
)
assert invocations_response.status_code == 200
# Verify all middlewares were executed
assert "X-Customer-Throttle" in invocations_response.headers
assert invocations_response.headers["X-Customer-Throttle"] == "applied"
assert "X-Customer-Processed" in invocations_response.headers
assert invocations_response.headers["X-Customer-Processed"] == "true"
# Verify input formatter was called
assert "X-Input-Formatter-Called" in invocations_response.headers
assert (
invocations_response.headers["X-Input-Formatter-Called"] == "true"
)
finally:
os.unlink(script_path)
@pytest.mark.asyncio
async def test_middleware_with_ping_endpoint(self):
"""Test that middlewares work with SageMaker ping endpoint."""
try:
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Customer writes a middleware script
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
f.write(
"""
from model_hosting_container_standards.common.fastapi.middleware import (
custom_middleware
)
@custom_middleware("pre_post_process")
async def ping_tracking_middleware(request, call_next):
response = await call_next(request)
if request.url.path == "/ping":
response.headers["X-Ping-Tracked"] = "true"
return response
"""
)
script_path = f.name
try:
script_dir = os.path.dirname(script_path)
script_name = os.path.basename(script_path)
env_vars = {
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
}
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--enforce-eager",
"--max-num-seqs",
"32",
]
with RemoteOpenAIServer(
MODEL_NAME_SMOLLM, args, env_dict=env_vars
) as server:
# Test ping endpoint with middleware
response = requests.get(server.url_for("ping"))
assert response.status_code == 200
assert "X-Ping-Tracked" in response.headers
assert response.headers["X-Ping-Tracked"] == "true"
finally:
os.unlink(script_path)
@pytest.mark.asyncio
async def test_middleware_env_var_override(self):
"""Test middleware environment variable overrides."""
try:
from model_hosting_container_standards.common.fastapi.config import (
FastAPIEnvVars,
)
from model_hosting_container_standards.sagemaker.config import (
SageMakerEnvVars,
)
except ImportError:
pytest.skip("model-hosting-container-standards not available")
# Create a script with middleware functions specified via env vars
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
f.write(
"""
from fastapi import Request
# Global flag to track if pre_process was called
_pre_process_called = False
async def env_throttle_middleware(request, call_next):
response = await call_next(request)
response.headers["X-Env-Throttle"] = "applied"
return response
async def env_pre_process(request: Request) -> Request:
# Mark that pre_process was called
global _pre_process_called
_pre_process_called = True
return request
async def env_post_process(response):
global _pre_process_called
if hasattr(response, 'headers'):
response.headers["X-Env-Post-Process"] = "applied"
# Since pre_process and post_process are combined into
# pre_post_process middleware,
# if post_process is called, pre_process should have been called too
if _pre_process_called:
response.headers["X-Pre-Process-Called"] = "true"
return response
"""
)
script_path = f.name
try:
script_dir = os.path.dirname(script_path)
script_name = os.path.basename(script_path)
# Set environment variables for middleware
# Use script_name with .py extension as per plugin example
env_vars = {
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
FastAPIEnvVars.CUSTOM_FASTAPI_MIDDLEWARE_THROTTLE: (
f"{script_name}:env_throttle_middleware"
),
FastAPIEnvVars.CUSTOM_PRE_PROCESS: f"{script_name}:env_pre_process",
FastAPIEnvVars.CUSTOM_POST_PROCESS: f"{script_name}:env_post_process",
}
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--enforce-eager",
"--max-num-seqs",
"32",
]
with RemoteOpenAIServer(
MODEL_NAME_SMOLLM, args, env_dict=env_vars
) as server:
response = requests.get(server.url_for("ping"))
assert response.status_code == 200
# Check if environment variable middleware was applied
headers = response.headers
# Verify that env var middlewares were applied
assert "X-Env-Throttle" in headers, (
"Throttle middleware should be applied via env var"
)
assert headers["X-Env-Throttle"] == "applied"
assert "X-Env-Post-Process" in headers, (
"Post-process middleware should be applied via env var"
)
assert headers["X-Env-Post-Process"] == "applied"
# Verify that pre_process was called
assert "X-Pre-Process-Called" in headers, (
"Pre-process should be called via env var"
)
assert headers["X-Pre-Process-Called"] == "true"
finally:
os.unlink(script_path)
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import openai # use the official client for correctness check
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from .conftest import (
HEADER_SAGEMAKER_CLOSED_SESSION_ID,
HEADER_SAGEMAKER_NEW_SESSION_ID,
HEADER_SAGEMAKER_SESSION_ID,
MODEL_NAME_SMOLLM,
)
CLOSE_BADREQUEST_CASES = [
(
"nonexistent_session_id",
{"session_id": "nonexistent-session-id"},
{},
"session not found",
),
("malformed_close_request", {}, {"extra-field": "extra-field-data"}, None),
]
@pytest.mark.asyncio
async def test_create_session_badrequest(basic_server_with_lora: RemoteOpenAIServer):
bad_response = requests.post(
basic_server_with_lora.url_for("invocations"),
json={"requestType": "NEW_SESSION", "extra-field": "extra-field-data"},
)
assert bad_response.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize(
"test_name,session_id_change,request_body_change,expected_error",
CLOSE_BADREQUEST_CASES,
)
async def test_close_session_badrequest(
basic_server_with_lora: RemoteOpenAIServer,
test_name: str,
session_id_change: dict[str, str],
request_body_change: dict[str, str],
expected_error: str | None,
):
# first attempt to create a session
url = basic_server_with_lora.url_for("invocations")
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
create_response.raise_for_status()
valid_session_id, expiration = create_response.headers.get(
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
).split(";")
assert valid_session_id
close_request_json = {"requestType": "CLOSE"}
if request_body_change:
close_request_json.update(request_body_change)
bad_session_id = session_id_change.get("session_id")
bad_close_response = requests.post(
url,
headers={HEADER_SAGEMAKER_SESSION_ID: bad_session_id or valid_session_id},
json=close_request_json,
)
# clean up created session, should succeed
clean_up_response = requests.post(
url,
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
json={"requestType": "CLOSE"},
)
clean_up_response.raise_for_status()
assert bad_close_response.status_code == 400
if expected_error:
assert expected_error in bad_close_response.json()["error"]["message"]
@pytest.mark.asyncio
async def test_close_session_invalidrequest(
basic_server_with_lora: RemoteOpenAIServer, async_client: openai.AsyncOpenAI
):
# first attempt to create a session
url = basic_server_with_lora.url_for("invocations")
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
create_response.raise_for_status()
valid_session_id, expiration = create_response.headers.get(
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
).split(";")
assert valid_session_id
close_request_json = {"requestType": "CLOSE"}
invalid_close_response = requests.post(
url,
# no headers to specify session_id
json=close_request_json,
)
# clean up created session, should succeed
clean_up_response = requests.post(
url,
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
json={"requestType": "CLOSE"},
)
clean_up_response.raise_for_status()
assert invalid_close_response.status_code == 424
assert "invalid session_id" in invalid_close_response.json()["error"]["message"]
@pytest.mark.asyncio
async def test_session(basic_server_with_lora: RemoteOpenAIServer):
# first attempt to create a session
url = basic_server_with_lora.url_for("invocations")
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
create_response.raise_for_status()
valid_session_id, expiration = create_response.headers.get(
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
).split(";")
assert valid_session_id
# test invocation with session id
request_args = {
"model": MODEL_NAME_SMOLLM,
"prompt": "what is 1+1?",
"max_completion_tokens": 5,
"temperature": 0.0,
"logprobs": False,
}
invocation_response = requests.post(
basic_server_with_lora.url_for("invocations"),
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
json=request_args,
)
invocation_response.raise_for_status()
# close created session, should succeed
close_response = requests.post(
url,
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
json={"requestType": "CLOSE"},
)
close_response.raise_for_status()
assert (
close_response.headers.get(HEADER_SAGEMAKER_CLOSED_SESSION_ID)
== valid_session_id
)
@@ -6,7 +6,7 @@ from unittest.mock import Mock
import pytest
import tests.utils as test_utils
from tests.utils import RemoteLaunchRenderServer, RemoteOpenAIServer
from tests.utils import RemoteOpenAIServer
@pytest.mark.parametrize(
@@ -30,7 +30,6 @@ def test_redact_sensitive_cli_arg_variants(args: list[str], expected: list[str])
("server_cls", "command"),
[
(RemoteOpenAIServer, ["vllm", "serve"]),
(RemoteLaunchRenderServer, ["vllm", "launch", "render"]),
],
)
def test_server_redacts_sensitive_values(