[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,245 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import anthropic
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--enable-prompt-tokens-details",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--served-model-name",
|
||||
"claude-3-7-sonnet-latest",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client_anthropic() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_messages(client: anthropic.AsyncAnthropic):
|
||||
resp = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "how are you!"}],
|
||||
)
|
||||
assert resp.stop_reason == "end_turn"
|
||||
assert resp.role == "assistant"
|
||||
|
||||
print(f"Anthropic response: {resp.model_dump_json()}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_message(client: anthropic.AsyncAnthropic):
|
||||
resp = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
system="you are a helpful assistant",
|
||||
messages=[{"role": "user", "content": "how are you!"}],
|
||||
)
|
||||
assert resp.stop_reason == "end_turn"
|
||||
assert resp.role == "assistant"
|
||||
|
||||
print(f"Anthropic response: {resp.model_dump_json()}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_streaming(client: anthropic.AsyncAnthropic):
|
||||
resp = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "how are you!"}],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
first_chunk = None
|
||||
chunk_count = 0
|
||||
async for chunk in resp:
|
||||
chunk_count += 1
|
||||
if first_chunk is None and chunk.type == "message_start":
|
||||
first_chunk = chunk
|
||||
print(chunk.model_dump_json())
|
||||
|
||||
assert chunk_count > 0
|
||||
assert first_chunk is not None, "message_start chunk was never observed"
|
||||
assert first_chunk.message is not None, "first chunk should include message"
|
||||
assert first_chunk.message.usage is not None, (
|
||||
"first chunk should include usage stats"
|
||||
)
|
||||
assert first_chunk.message.usage.output_tokens == 0
|
||||
assert first_chunk.message.usage.input_tokens > 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic):
|
||||
resp = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather like in New York today?"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"description": "Useful for querying the weather in a specified city.",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "City or region, for example: "
|
||||
"New York, London, Tokyo, etc.",
|
||||
}
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
}
|
||||
],
|
||||
stream=False,
|
||||
)
|
||||
assert resp.stop_reason == "tool_use"
|
||||
assert resp.role == "assistant"
|
||||
|
||||
print(f"Anthropic response: {resp.model_dump_json()}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
|
||||
resp = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in New York today?",
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"description": "Useful for querying the weather in a specified city.",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "City or region, for example: "
|
||||
"New York, London, Tokyo, etc.",
|
||||
}
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
async for chunk in resp:
|
||||
print(chunk.model_dump_json())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic):
|
||||
response = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email:"
|
||||
"John Smith (john@example.com) is interested in our "
|
||||
"Enterprise plan and wants to schedule a demo for next Tuesday at 2pm.",
|
||||
}
|
||||
],
|
||||
output_config={
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"},
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
print(response.content[0].text)
|
||||
json_obj = json.loads(response.content[0].text)
|
||||
for key in ["name", "email", "plan_interest", "demo_requested"]:
|
||||
assert key in json_obj, f"Missing key in output: {key}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic):
|
||||
async def get_stream_usage(resp):
|
||||
prompt_tokens = None
|
||||
usage = None
|
||||
async for chunk in resp:
|
||||
if (
|
||||
chunk.type == "message_start"
|
||||
and chunk.message is not None
|
||||
and chunk.message.usage is not None
|
||||
):
|
||||
prompt_tokens = chunk.message.usage.input_tokens
|
||||
elif chunk.type == "message_delta" and chunk.usage is not None:
|
||||
usage = chunk.usage
|
||||
|
||||
assert usage is not None
|
||||
assert usage.input_tokens >= 0
|
||||
assert usage.output_tokens >= 0
|
||||
cache_created = usage.cache_creation_input_tokens
|
||||
cache_read = usage.cache_read_input_tokens
|
||||
assert cache_read is not None
|
||||
assert cache_created is not None
|
||||
assert cache_created >= 0
|
||||
assert cache_read >= 0
|
||||
assert prompt_tokens == usage.input_tokens + cache_created + cache_read
|
||||
return usage
|
||||
|
||||
request = dict(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1,
|
||||
extra_body={"temperature": 0.0},
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Cache coverage sentinel. " * 256
|
||||
+ "Answer with exactly one word: ok.",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
cold_usage = await get_stream_usage(await client.messages.create(**request))
|
||||
assert cold_usage.cache_read_input_tokens == 0
|
||||
assert cold_usage.cache_creation_input_tokens is not None
|
||||
assert cold_usage.cache_creation_input_tokens > 0
|
||||
|
||||
warm_usage = await get_stream_usage(await client.messages.create(**request))
|
||||
assert warm_usage.cache_read_input_tokens is not None
|
||||
assert warm_usage.cache_read_input_tokens > 0
|
||||
@@ -1,52 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Anthropic protocol exports used by serving.
|
||||
|
||||
Guards against Docker/nightly images shipping a stale protocol module that is
|
||||
missing symbols imported by ``vllm.entrypoints.anthropic.serving`` (issue #44759).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicContentBlock,
|
||||
AnthropicContextManagement,
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicDelta,
|
||||
AnthropicError,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
SERVING_PROTOCOL_EXPORTS = (
|
||||
AnthropicContentBlock,
|
||||
AnthropicContextManagement,
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicDelta,
|
||||
AnthropicError,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
|
||||
def test_serving_protocol_exports_are_importable():
|
||||
for export in SERVING_PROTOCOL_EXPORTS:
|
||||
assert export is not None
|
||||
|
||||
|
||||
def test_anthropic_output_config_instantiation():
|
||||
config = AnthropicOutputConfig()
|
||||
assert config.effort is None
|
||||
assert config.format is None
|
||||
@@ -1,693 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``vllm/entrypoints/cohere/api_router.py``.
|
||||
|
||||
Covers:
|
||||
|
||||
* The optional-import guard: ``attach_router`` is a no-op when the
|
||||
``cohere`` SDK isn't installed.
|
||||
* The env-var opt-in gate: ``attach_router`` is a no-op unless
|
||||
``VLLM_ENABLE_COHERE_API=1`` is set.
|
||||
* The router wiring: response shapes (JSON + SSE), error translation,
|
||||
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
|
||||
Implemented).
|
||||
* The render route (``POST /cohere/v2/chat/render``): that the v2 body
|
||||
reaches :class:`ServingRender` as a converted ``ChatCompletionRequest``
|
||||
and that the resulting ``GenerateRequest`` is returned verbatim.
|
||||
"""
|
||||
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncGenerator
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.cohere import api_router as api_router_mod
|
||||
from vllm.entrypoints.cohere.api_router import attach_router
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CohereChatV2Response,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import (
|
||||
_API_SERVER_ONLY_TEMPLATE_KWARGS,
|
||||
CohereServingChatV2,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
||||
http_exception_handler,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||
validation_exception_handler,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import (
|
||||
vllm_error_handler,
|
||||
)
|
||||
from vllm.exceptions import VLLMError, VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_cohere_api(monkeypatch):
|
||||
"""Auto-enable the Cohere API gate for every test in this module.
|
||||
|
||||
The endpoint is opt-in in production (``VLLM_ENABLE_COHERE_API=1``);
|
||||
every test in this file exercises the enabled path *except* the
|
||||
dedicated gate test in :class:`TestEnvVarGate`, which unsets the
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Fakes
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class _Handler:
|
||||
"""Minimal stand-in for :class:`CohereServingChatV2` used by the
|
||||
router. Each test sets ``self.result`` to either:
|
||||
|
||||
* a :class:`CohereChatV2Response` (non-streaming JSON path);
|
||||
* an async generator yielding SSE frames (streaming path);
|
||||
* an :class:`ErrorResponse` (error envelope path); or
|
||||
* an exception (router-level error path).
|
||||
"""
|
||||
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
|
||||
async def create_chat_v2(self, request, raw_request):
|
||||
if isinstance(self.result, Exception):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
class _RenderChatHandler:
|
||||
"""Stand-in for :class:`CohereServingChatV2` on the render route.
|
||||
|
||||
Only ``to_chat_completion_request`` is reachable from
|
||||
``/cohere/v2/chat/render``, and it delegates to the real conversion
|
||||
classmethod (which needs no engine or tokenizer) so the tests
|
||||
exercise the actual v2 -> ``ChatCompletionRequest`` translation
|
||||
rather than a stub of it.
|
||||
"""
|
||||
|
||||
def to_chat_completion_request(self, request):
|
||||
return CohereServingChatV2._convert_v2_to_chat_completion(request)
|
||||
|
||||
def _engine_chat_template_kwargs(self, chat_template_kwargs):
|
||||
return CohereServingChatV2._engine_chat_template_kwargs(
|
||||
self, # type: ignore[arg-type]
|
||||
chat_template_kwargs,
|
||||
)
|
||||
|
||||
|
||||
class _RenderHandler:
|
||||
"""Minimal stand-in for :class:`ServingRender`.
|
||||
|
||||
Records the ``ChatCompletionRequest`` it was handed so tests can
|
||||
assert on the conversion, and returns ``self.result`` - either a
|
||||
:class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception
|
||||
to raise.
|
||||
"""
|
||||
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.seen_request = None
|
||||
|
||||
async def render_chat_request(self, request):
|
||||
self.seen_request = request
|
||||
if isinstance(self.result, Exception):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
def _generate_request() -> GenerateRequest:
|
||||
return GenerateRequest(
|
||||
request_id="chatcmpl-abc",
|
||||
token_ids=[5, 6, 7],
|
||||
sampling_params=SamplingParams(temperature=0.25),
|
||||
model="m",
|
||||
)
|
||||
|
||||
|
||||
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_render_app(
|
||||
chat_handler, render_handler, *, enable_scale_out: bool = True
|
||||
) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = chat_handler
|
||||
app.state.serving_render = render_handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
|
||||
"""Build a FastAPI app that mirrors the real vLLM setup by installing
|
||||
validation, HTTP, and vLLM exception handlers. The
|
||||
:class:`CohereErrorEnvelopeMiddleware` registered by ``attach_router``
|
||||
is expected to translate any resulting vLLM ``ErrorResponse`` body
|
||||
into the ``CohereError`` wire shape.
|
||||
"""
|
||||
app = FastAPI()
|
||||
# ``validation_exception_handler`` reads ``req.app.state.args``; the
|
||||
# real cli builds this via argparse.
|
||||
app.state.args = Namespace(
|
||||
log_error_stack=False, enable_scale_out=True, tokens_only=False
|
||||
)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
app.exception_handler(VLLMError)(vllm_error_handler)
|
||||
return app
|
||||
|
||||
|
||||
def _minimal_request_body() -> dict:
|
||||
return {
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Optional-import guard
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOptionalCohereImport:
|
||||
"""``attach_router`` probes for the SDK once at module load (because
|
||||
the route handler uses types imported from ``cohere``) and stashes
|
||||
the result in ``_SDK_AVAILABLE``. Tests simulate the "SDK missing"
|
||||
state by flipping that flag for the duration of the test.
|
||||
|
||||
``attach_router`` checks the env-var gate before the SDK probe, so
|
||||
the SDK-missing branch is only reachable when the operator opts in
|
||||
via ``VLLM_ENABLE_COHERE_API=1``. The flag-off-and-SDK-missing case
|
||||
below exists to pin down that ordering — the flag-off short-circuits
|
||||
"""
|
||||
|
||||
def test_flag_off_and_sdk_missing_stays_silent_about_sdk(self, monkeypatch, caplog):
|
||||
"""Flag off doesn't do SDK-missing check.
|
||||
|
||||
When the operator hasn't opted in, ``attach_router`` must not
|
||||
warn about the ``cohere`` SDK being missing: they never asked
|
||||
for the endpoint, so surfacing the SDK gap is misleading noise.
|
||||
Only the flag-off DEBUG message should fire.
|
||||
"""
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
|
||||
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
# The flag-off short-circuit ran; the SDK check never did.
|
||||
assert not any(
|
||||
"SDK is not installed" in rec.message for rec in caplog.records
|
||||
), "SDK-missing log leaked despite the flag being off"
|
||||
|
||||
def test_flag_on_but_sdk_missing_logs_warning(self, monkeypatch, caplog):
|
||||
"""Misconfiguration path: the operator explicitly opted into the
|
||||
endpoint via ``VLLM_ENABLE_COHERE_API=1`` (already set by the
|
||||
autouse fixture) but forgot to install ``cohere``.
|
||||
"""
|
||||
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
|
||||
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
warn_sdk_records = [
|
||||
rec
|
||||
for rec in caplog.records
|
||||
if "VLLM_ENABLE_COHERE_API=1" in rec.message
|
||||
and "SDK is not installed" in rec.message
|
||||
]
|
||||
assert warn_sdk_records, (
|
||||
"expected a WARNING that pairs the opt-in flag with the "
|
||||
"missing SDK so operators notice the misconfiguration"
|
||||
)
|
||||
assert all(rec.levelname == "WARNING" for rec in warn_sdk_records)
|
||||
|
||||
def test_attach_router_registers_route_when_cohere_present(self):
|
||||
app = _build_app(handler=None)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# VLLM_ENABLE_COHERE_API gate
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarGate:
|
||||
"""The Cohere v2 endpoint is opt-in via ``VLLM_ENABLE_COHERE_API``.
|
||||
|
||||
Even with the SDK installed, :func:`attach_router` must skip route
|
||||
registration and middleware installation unless the env flag is
|
||||
set. The autouse fixture on this module enables the flag by
|
||||
default, so each test here explicitly disables it.
|
||||
"""
|
||||
|
||||
def test_attach_router_noop_when_flag_unset(self, monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
|
||||
# The flag-off skip logs at DEBUG on purpose: this is the default
|
||||
# state for every non-Cohere vLLM deployment, so an INFO log on
|
||||
# every server startup would be pointless noise. The test raises
|
||||
# caplog's level accordingly.
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
debug_flag_records = [
|
||||
rec
|
||||
for rec in caplog.records
|
||||
if "VLLM_ENABLE_COHERE_API is not set" in rec.message
|
||||
]
|
||||
assert debug_flag_records, (
|
||||
"expected a DEBUG message that the cohere flag is off"
|
||||
)
|
||||
assert all(rec.levelname == "DEBUG" for rec in debug_flag_records)
|
||||
|
||||
def test_attach_router_noop_when_flag_zero(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "0")
|
||||
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Endpoint behavior
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEndpoint:
|
||||
def test_501_when_handler_missing(self):
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
body = r.json()
|
||||
assert "does not support" in body["message"]
|
||||
assert "id" not in body # excluded by ``exclude_none=True``
|
||||
|
||||
def test_non_streaming_response_is_json(self):
|
||||
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hello"}])
|
||||
result = CohereChatV2Response(id="r1", finish_reason="COMPLETE", message=msg)
|
||||
app = _build_app(handler=_Handler(result))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
body = r.json()
|
||||
assert body["id"] == "r1"
|
||||
assert body["finish_reason"] == "COMPLETE"
|
||||
assert body["message"]["content"][0]["text"] == "hello"
|
||||
|
||||
def test_streaming_response_is_sse(self):
|
||||
async def _gen() -> AsyncGenerator[str, None]:
|
||||
yield 'data: {"type":"message-start"}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
app = _build_app(handler=_Handler(_gen()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={**_minimal_request_body(), "stream": True},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
body = r.text
|
||||
assert "message-start" in body
|
||||
assert body.rstrip().endswith("[DONE]")
|
||||
|
||||
def test_error_response_translated_to_cohere_envelope(self):
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message="bad request",
|
||||
type="bad_request",
|
||||
code=400,
|
||||
)
|
||||
)
|
||||
app = _build_app(handler=_Handler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
assert body == {"message": "bad request"}
|
||||
|
||||
def test_handler_exception_returns_500_envelope(self):
|
||||
app = _build_app(handler=_Handler(RuntimeError("kaboom")))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
body = r.json()
|
||||
assert body == {"message": "kaboom"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
VLLMValidationError("bad temperature", parameter="temperature"),
|
||||
ValueError("bad temperature"),
|
||||
],
|
||||
ids=["vllm_validation_error", "value_error"],
|
||||
)
|
||||
def test_handler_client_error_returns_400_envelope(self, exc):
|
||||
"""Client-caused handler errors are 4xx, matching the OpenAI routes."""
|
||||
app = _build_app(handler=_Handler(exc))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert "bad temperature" in r.json()["message"]
|
||||
|
||||
def test_non_json_content_type_rejected(self):
|
||||
"""The ``validate_json_request`` dependency raises
|
||||
``RequestValidationError`` (HTTP 422) for non-JSON content
|
||||
types, matching the behavior of the other vLLM API routers.
|
||||
"""
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
content=json.dumps(_minimal_request_body()),
|
||||
headers={"content-type": "text/plain"},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
def test_invalid_body_returns_422(self):
|
||||
# ``model`` is required; omit it to trip Pydantic validation.
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Render endpoint
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRenderEndpoint:
|
||||
"""``POST /cohere/v2/chat/render`` is the Cohere counterpart to
|
||||
``POST /v1/chat/completions/render``: it converts the v2 body to a
|
||||
``ChatCompletionRequest`` and hands it to the shared
|
||||
``ServingRender``, returning the resulting ``GenerateRequest``
|
||||
instead of running generation.
|
||||
|
||||
The route therefore spans *two* app-state handlers, and these tests
|
||||
pin both halves: that the conversion preserves the Cohere-specific
|
||||
request surface, and that the render handler's output (or error)
|
||||
reaches the client unchanged.
|
||||
"""
|
||||
|
||||
def test_route_registered(self):
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self):
|
||||
app = _build_app(handler=None, enable_scale_out=False)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
def test_returns_generate_request_json(self):
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
app = _build_render_app(_RenderChatHandler(), render_handler)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
body = r.json()
|
||||
assert body["request_id"] == "chatcmpl-abc"
|
||||
assert body["token_ids"] == [5, 6, 7]
|
||||
assert body["sampling_params"]["temperature"] == 0.25
|
||||
# The body must be a wire-valid GenerateRequest so callers can
|
||||
# feed it straight back to /v1/generate.
|
||||
assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7]
|
||||
|
||||
def test_cohere_request_surface_survives_conversion(self):
|
||||
"""The point of routing through ``CohereServingChatV2`` rather
|
||||
than asking callers to pre-convert: Cohere-only fields
|
||||
(``documents``, ``safety_mode``, ``citation_options``) must land
|
||||
in ``chat_template_kwargs`` so the rendered prompt matches what
|
||||
``/cohere/v2/chat`` would have produced.
|
||||
"""
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
app = _build_render_app(_RenderChatHandler(), render_handler)
|
||||
body = {
|
||||
**_minimal_request_body(),
|
||||
"documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}],
|
||||
"safety_mode": "STRICT",
|
||||
"max_tokens": 32,
|
||||
}
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=body)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
|
||||
seen = render_handler.seen_request
|
||||
assert seen is not None
|
||||
assert seen.model == "m"
|
||||
assert seen.messages[0]["content"] == "hi"
|
||||
# ``max_tokens`` matters because ServingRender feeds it to
|
||||
# ``get_max_tokens`` when building sampling params.
|
||||
assert seen.max_completion_tokens == 32
|
||||
kwargs = seen.chat_template_kwargs or {}
|
||||
assert kwargs["documents"][0]["id"] == "doc-1"
|
||||
# The conversion lower-cases safety_mode for the renderer.
|
||||
assert kwargs["safety_mode"] == "strict"
|
||||
|
||||
def test_api_server_only_kwargs_not_forwarded(self):
|
||||
"""As on ``/cohere/v2/chat``, the engine's reasoning parser must not
|
||||
receive the template kwargs that only the API server reads."""
|
||||
result = _generate_request()
|
||||
result.reasoning_parser_kwargs = ReasoningParserKwargs(
|
||||
chat_template_kwargs={
|
||||
"safety_mode": "strict",
|
||||
**dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"),
|
||||
}
|
||||
)
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(result))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.json()["reasoning_parser_kwargs"] == {
|
||||
"chat_template_kwargs": {"safety_mode": "strict"}
|
||||
}
|
||||
|
||||
def test_501_when_chat_handler_missing(self):
|
||||
app = _build_render_app(None, _RenderHandler(_generate_request()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
assert "does not support" in r.json()["message"]
|
||||
|
||||
def test_501_when_render_handler_missing(self):
|
||||
# A server without ``serving_render`` in app state (e.g. a
|
||||
# non-generate task set) can't render, even though the Cohere
|
||||
# chat handler exists.
|
||||
app = _build_render_app(_RenderChatHandler(), None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
assert "does not support" in r.json()["message"]
|
||||
|
||||
def test_error_response_translated_to_cohere_envelope(self):
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(message="unknown model", type="NotFound", code=404)
|
||||
)
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_FOUND
|
||||
assert r.json() == {"message": "unknown model"}
|
||||
|
||||
def test_render_exception_returns_500_envelope(self):
|
||||
app = _build_render_app(
|
||||
_RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded"))
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
assert r.json() == {"message": "tokenizer exploded"}
|
||||
|
||||
def test_conversion_client_error_returns_400_envelope(self):
|
||||
"""Conversion runs inside the same ``try`` as the render call, so
|
||||
a bad v2 body that slips past Pydantic still yields a 4xx in the
|
||||
Cohere error envelope rather than an unhandled 500 with vLLM's shape.
|
||||
"""
|
||||
|
||||
class _Boom:
|
||||
def to_chat_completion_request(self, request):
|
||||
raise ValueError("unconvertible message")
|
||||
|
||||
app = _build_render_app(_Boom(), _RenderHandler(_generate_request()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert r.json() == {"message": "unconvertible message"}
|
||||
|
||||
def test_invalid_body_returns_422(self):
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={"messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# CohereErrorEnvelopeMiddleware
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCohereErrorEnvelope:
|
||||
"""When the app installs vLLM's global exception handlers, validation
|
||||
and HTTP errors escape as ``ErrorResponse`` bodies before the route
|
||||
handler runs. The middleware installed by ``attach_router`` must
|
||||
normalise those bodies to the ``CohereError`` shape declared on the
|
||||
endpoint's OpenAPI ``responses`` map so schema-conformance tests
|
||||
(``test_openai_schema.py``) don't see a mismatch on ``/cohere/*``
|
||||
responses.
|
||||
"""
|
||||
|
||||
def test_validation_error_body_is_cohere_shaped(self):
|
||||
# ``model=""`` and ``messages=[]`` trip our custom field
|
||||
# validators, which raise VLLMValidationError and are routed
|
||||
# through ``vllm_error_handler`` in the real vLLM
|
||||
# server (producing the ``{"error": {...}}`` shape).
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json={"messages": [], "model": ""})
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# ``CohereError`` has ``message`` at the top level, not nested
|
||||
# under an ``error`` envelope.
|
||||
assert "error" not in body
|
||||
assert "message" in body
|
||||
assert isinstance(body["message"], str) and body["message"]
|
||||
|
||||
@pytest.mark.parametrize("path", ["/cohere/v2/chat", "/cohere/v2/chat/render"])
|
||||
def test_required_tool_choice_without_tools_returns_400(self, path):
|
||||
"""Reject an impossible tool choice before reaching either handler."""
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
path,
|
||||
json={**_minimal_request_body(), "tool_choice": "REQUIRED"},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = response.json()
|
||||
assert "error" not in body
|
||||
assert "REQUIRED" in body["message"]
|
||||
assert "tools" in body["message"]
|
||||
|
||||
def test_http_error_body_is_cohere_shaped(self):
|
||||
# A raised ``HTTPException`` from anywhere in the request cycle
|
||||
# is routed through ``http_exception_handler`` (producing the
|
||||
# ``ErrorResponse`` shape) and must be translated.
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
|
||||
@app.get("/cohere/v2/boom")
|
||||
async def _boom():
|
||||
raise HTTPException(status_code=418, detail="teapot")
|
||||
|
||||
with TestClient(app) as client:
|
||||
r = client.get("/cohere/v2/boom")
|
||||
assert r.status_code == 418
|
||||
body = r.json()
|
||||
assert body == {"message": "teapot"}
|
||||
|
||||
def test_non_cohere_path_is_not_translated(self):
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
|
||||
@app.get("/v1/other")
|
||||
async def _other():
|
||||
raise HTTPException(status_code=400, detail="nope")
|
||||
|
||||
with TestClient(app) as client:
|
||||
r = client.get("/v1/other")
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# Non-cohere paths keep the vLLM ``ErrorResponse`` shape.
|
||||
assert "error" in body
|
||||
assert body["error"]["message"] == "nope"
|
||||
|
||||
def test_streaming_response_passes_through(self):
|
||||
# SSE responses have content-type text/event-stream; the
|
||||
# middleware must never buffer these (which would break
|
||||
# streaming) even though they're on ``/cohere/*``.
|
||||
async def _gen() -> AsyncGenerator[str, None]:
|
||||
yield 'data: {"type":"message-start"}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
app = _build_app_with_vllm_handlers(handler=_Handler(_gen()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={**_minimal_request_body(), "stream": True},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
assert "message-start" in r.text
|
||||
assert r.text.rstrip().endswith("[DONE]")
|
||||
|
||||
def test_already_cohere_shaped_body_passes_through(self):
|
||||
# When the handler returns an ``ErrorResponse`` the route
|
||||
# itself translates it to ``CohereError``; the middleware sees
|
||||
# the ``CohereError`` shape and must leave it alone.
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(message="already cohere", type="Bad Request", code=400)
|
||||
)
|
||||
app = _build_app_with_vllm_handlers(handler=_Handler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# No ``error`` wrapper: the route already emitted the wire shape.
|
||||
assert body == {"message": "already cohere"}
|
||||
|
||||
def test_request_id_preserved_in_translated_body(self):
|
||||
# Client-provided ``X-Request-Id`` should be echoed as
|
||||
# ``CohereError.id`` so callers can correlate failures.
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": [], "model": ""},
|
||||
headers={"X-Request-Id": "req-abc"},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
assert body.get("id") == "req-abc"
|
||||
@@ -1,343 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end integration tests for the Cohere Chat v2 endpoints
|
||||
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
|
||||
|
||||
These tests spin up a real ``vllm serve`` process via
|
||||
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
|
||||
contract over HTTP. They mirror the pattern used by
|
||||
:mod:`tests.entrypoints.anthropic.test_messages`.
|
||||
|
||||
Two layers of integration are covered:
|
||||
|
||||
1. Raw HTTP via :mod:`httpx` — always runs, verifies the wire contract.
|
||||
2. Cohere SDK (``pip install cohere``) — auto-skipped when the optional
|
||||
dependency isn't installed, verifies SDK-level interop.
|
||||
|
||||
We use a tiny generic chat model (``HuggingFaceTB/SmolLM2-135M-Instruct``,
|
||||
≈135M params) so the fixture stays runnable on CPU-only laptops. The
|
||||
Cohere v2 endpoint is model-agnostic and just performs the v2 ↔ OpenAI
|
||||
chat-completion translation, so the choice of model only matters for
|
||||
response *shape* — the translation logic itself is unit-tested
|
||||
elsewhere.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# Tiny chat-tuned model so the fixture is cheap to boot on CPU-only
|
||||
# hosts (Mac, CI runners without GPUs). The Cohere v2 translation is
|
||||
# completely independent of which underlying chat model is loaded.
|
||||
MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct"
|
||||
SERVED_MODEL_NAME = "command-r-plus-08-2024"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
# 1k tokens is plenty for the prompts these tests send; trimming
|
||||
# it from the default keeps the KV-cache footprint tiny.
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
# Advertise a Cohere model name so Cohere SDK ``model=`` calls
|
||||
# round-trip without the ``model not found`` check.
|
||||
"--served-model-name",
|
||||
SERVED_MODEL_NAME,
|
||||
# Disable the reasoning-model path so the test doesn't require
|
||||
# the conversation to surface a thinking block (SmolLM2 doesn't
|
||||
# emit Cohere-style reasoning tokens out of the box).
|
||||
"--no-cohere-is-reasoning-model",
|
||||
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
|
||||
# startup. The default (4 GB) trips Mac's RAM check on smaller
|
||||
# machines; 1 GB is more than enough for max_model_len=1024 and
|
||||
# max_num_seqs=4.
|
||||
# ``VLLM_ENABLE_COHERE_API=1`` flips the opt-in gate that
|
||||
# ``vllm.entrypoints.cohere.api_router.attach_router`` reads to
|
||||
# decide whether to register ``POST /cohere/v2/chat`` at server
|
||||
# startup. Without it the fixture would boot a server that returns
|
||||
# 404 for every request the tests below make.
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Layer 1: raw HTTP contract (no optional deps)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def httpx_client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_root, timeout=httpx.Timeout(120.0)
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_non_streaming(httpx_client: httpx.AsyncClient):
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Say hi."}],
|
||||
"max_tokens": 16,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
# The response envelope follows Cohere v2's schema.
|
||||
assert "message" in payload
|
||||
assert payload["message"]["role"] == "assistant"
|
||||
# ``content`` is a list of content blocks; the synthesized
|
||||
# ``CohereServingChatV2`` should always emit at least one ``text``
|
||||
# block (it falls back to an empty block when the model returned
|
||||
# nothing).
|
||||
content = payload["message"]["content"]
|
||||
assert isinstance(content, list) and len(content) >= 1
|
||||
assert content[0]["type"] == "text"
|
||||
# ``usage`` is always populated by the translator.
|
||||
assert "usage" in payload
|
||||
assert "billed_units" in payload["usage"]
|
||||
assert "tokens" in payload["usage"]
|
||||
# ``finish_reason`` is one of Cohere's enum values.
|
||||
assert payload["finish_reason"] in {
|
||||
"COMPLETE",
|
||||
"MAX_TOKENS",
|
||||
"STOP_SEQUENCE",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_streaming(httpx_client: httpx.AsyncClient):
|
||||
"""Streaming returns SSE frames in the v2 message-lifecycle shape."""
|
||||
events: list[dict] = []
|
||||
async with httpx_client.stream(
|
||||
"POST",
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"max_tokens": 8,
|
||||
"stream": True,
|
||||
},
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
assert resp.headers["content-type"].startswith("text/event-stream")
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
data = line[len("data: ") :]
|
||||
if data == "[DONE]":
|
||||
events.append({"type": "_DONE_"})
|
||||
continue
|
||||
events.append(json.loads(data))
|
||||
|
||||
types = [ev["type"] for ev in events]
|
||||
# The lifecycle always starts with message-start and ends with [DONE]
|
||||
# preceded by message-end.
|
||||
assert types[0] == "message-start"
|
||||
assert types[-1] == "_DONE_"
|
||||
assert types[-2] == "message-end"
|
||||
# message-start must carry the chunk/message id.
|
||||
assert events[0].get("id")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_validation_error_returns_400(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
# Missing required ``model`` field → FastAPI/Pydantic returns 400.
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": []},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
|
||||
"""``/cohere/v2/chat/render`` tokenizes without running the model.
|
||||
|
||||
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
|
||||
contract that makes the endpoint useful: v2 sampling fields
|
||||
(``temperature``, ``stop_sequences``) survive the v2 ->
|
||||
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
|
||||
render once and generate later without re-deriving them.
|
||||
"""
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Say hi."}],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.125,
|
||||
"stop_sequences": ["END"],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
|
||||
token_ids = data["token_ids"]
|
||||
assert isinstance(token_ids, list) and len(token_ids) > 0
|
||||
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
|
||||
assert data["request_id"]
|
||||
|
||||
sampling_params = data["sampling_params"]
|
||||
assert sampling_params["temperature"] == 0.125
|
||||
assert sampling_params["stop"] == ["END"]
|
||||
|
||||
# No generation happened, so the response carries none of the chat
|
||||
# envelope the /cohere/v2/chat endpoint would return.
|
||||
assert "message" not in data
|
||||
assert "finish_reason" not in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""The rendered ids are the real templated prompt, not a debug dump.
|
||||
|
||||
Round-tripping them through ``/detokenize`` and finding the user
|
||||
message back proves the endpoint applied the chat template and
|
||||
tokenized it, which is what a caller splitting prompt construction
|
||||
from generation depends on.
|
||||
"""
|
||||
render_resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"max_tokens": 8,
|
||||
},
|
||||
)
|
||||
assert render_resp.status_code == 200, render_resp.text
|
||||
token_ids = render_resp.json()["token_ids"]
|
||||
|
||||
detok_resp = await httpx_client.post(
|
||||
"/detokenize",
|
||||
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
|
||||
)
|
||||
assert detok_resp.status_code == 200, detok_resp.text
|
||||
prompt = detok_resp.json()["prompt"]
|
||||
assert "What is the capital of France?" in prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_validation_error_returns_400(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""Bad bodies get the Cohere error envelope, not vLLM's."""
|
||||
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
|
||||
assert resp.status_code == 400
|
||||
body = resp.json()
|
||||
assert "error" not in body
|
||||
assert body.get("message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_documents_field_accepted(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""The v2 endpoint forwards ``documents`` into chat_template_kwargs.
|
||||
|
||||
We only assert the request is accepted and produces a 200 response —
|
||||
the renderer-level effect is covered by ``tests/renderers/test_cohere.py``.
|
||||
"""
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Summarize."}],
|
||||
"documents": [{"id": "d1", "data": {"title": "T", "snippet": "S"}}],
|
||||
"max_tokens": 16,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Layer 2: Cohere SDK round-trip (auto-skipped if SDK absent)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def cohere_async_client(server):
|
||||
cohere = pytest.importorskip("cohere")
|
||||
# The vLLM endpoint is mounted at ``/cohere/v2/chat`` while the
|
||||
# cohere SDK targets ``${base_url}/v2/chat``; point base_url at the
|
||||
# ``/cohere`` prefix so paths line up.
|
||||
client = cohere.AsyncClientV2(
|
||||
api_key="dummy",
|
||||
base_url=server.url_for("cohere"),
|
||||
)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
# ``AsyncClientV2`` exposes a sync close; if a future version
|
||||
# adds aclose we still close cleanly.
|
||||
close = getattr(client, "aclose", None) or getattr(client, "close", None)
|
||||
if close is not None:
|
||||
result = close()
|
||||
if hasattr(result, "__await__"):
|
||||
await result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_sdk_non_streaming(cohere_async_client):
|
||||
resp = await cohere_async_client.chat(
|
||||
model=SERVED_MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Say hi."}],
|
||||
max_tokens=16,
|
||||
)
|
||||
# SDK parses our JSON into typed objects.
|
||||
assert resp.message.role == "assistant"
|
||||
assert resp.message.content is not None
|
||||
assert len(resp.message.content) >= 1
|
||||
assert resp.message.content[0].type == "text"
|
||||
assert resp.finish_reason in {
|
||||
"COMPLETE",
|
||||
"MAX_TOKENS",
|
||||
"STOP_SEQUENCE",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_sdk_streaming(cohere_async_client):
|
||||
events: list[str] = []
|
||||
stream = cohere_async_client.chat_stream(
|
||||
model=SERVED_MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
max_tokens=8,
|
||||
)
|
||||
async for ev in stream:
|
||||
events.append(ev.type)
|
||||
|
||||
assert events, "SDK stream yielded no events"
|
||||
assert events[0] == "message-start"
|
||||
assert events[-1] == "message-end"
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for :mod:`vllm.entrypoints.cohere.cohere_chat_message`.
|
||||
|
||||
The module hosts the citation-carrying subclasses of the OpenAI chat
|
||||
completion protocol (``CohereChatMessage`` / ``CohereDeltaMessage``) plus
|
||||
the shared ``Citation`` / ``CitationSource`` shapes. These tests pin:
|
||||
|
||||
1. Serialization behavior we own (empty-``citations`` cleanup on the
|
||||
subclass ``_serialize``, ``exclude_none`` shape, round-trip).
|
||||
2. The constraint that ``type`` discriminators reject unknown values.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation,
|
||||
CitationSource,
|
||||
CohereChatMessage,
|
||||
CohereDeltaMessage,
|
||||
)
|
||||
|
||||
# ======================================================================
|
||||
# CitationSource
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCitationSource:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CitationSource(type="other")
|
||||
|
||||
def test_none_fields_excluded_from_dump(self):
|
||||
s = CitationSource(type="document", id="d1")
|
||||
assert s.model_dump(exclude_none=True) == {
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
}
|
||||
|
||||
def test_resolved_source_survives_json_round_trip(self):
|
||||
# The parser produces fully-resolved sources (see
|
||||
# ``_melody_sources_to_vllm`` in
|
||||
# ``vllm/reasoning/cohere_command_reasoning_parser.py``) which
|
||||
# then flow through the internal streaming pipeline (parser ->
|
||||
# OpenAI stream chunk -> ``_chat_completion_stream_to_v2``).
|
||||
# Every field must round-trip through ``model_dump_json`` at
|
||||
# that layer or the wire event ends up missing pieces.
|
||||
src = CitationSource(
|
||||
type="tool",
|
||||
id="res_a0",
|
||||
tool_output={"id": "res_a0", "text": "r"},
|
||||
)
|
||||
src2 = CitationSource.model_validate_json(src.model_dump_json())
|
||||
assert src2.type == "tool"
|
||||
assert src2.id == "res_a0"
|
||||
assert src2.tool_output == {"id": "res_a0", "text": "r"}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Citation
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCitation:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
Citation(type="OTHER")
|
||||
|
||||
def test_dump_excludes_none_fields(self):
|
||||
c = Citation(start=0, end=5, text="hello")
|
||||
dumped = c.model_dump(exclude_none=True)
|
||||
assert dumped == {
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "hello",
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereDeltaMessage
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereDeltaMessage:
|
||||
def test_citations_omitted_from_dump_when_none(self):
|
||||
# Non-grounded deltas should never carry ``citations: null`` on
|
||||
# the wire even when routed through the Cohere subclass.
|
||||
d = CohereDeltaMessage(content="hello")
|
||||
dumped = d.model_dump(exclude_none=True)
|
||||
assert "citations" not in dumped
|
||||
|
||||
def test_citations_dump_round_trip(self):
|
||||
d = CohereDeltaMessage(citations=[Citation(start=0, end=5, text="hi")])
|
||||
dumped = d.model_dump(exclude_none=True)
|
||||
assert dumped["citations"][0]["text"] == "hi"
|
||||
# Round-trip through the subclass to confirm the field validates.
|
||||
d2 = CohereDeltaMessage.model_validate(dumped)
|
||||
assert d2.citations[0].text == "hi"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatMessage
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatMessage:
|
||||
def test_citations_omitted_from_dump_when_none(self):
|
||||
m = CohereChatMessage(role="assistant", content="hi")
|
||||
dumped = m.model_dump(exclude_none=True)
|
||||
assert "citations" not in dumped
|
||||
|
||||
def test_citations_dump_round_trip(self):
|
||||
m = CohereChatMessage(
|
||||
role="assistant",
|
||||
content="hello",
|
||||
citations=[
|
||||
Citation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[CitationSource(type="document", id="d1")],
|
||||
)
|
||||
],
|
||||
)
|
||||
dumped = m.model_dump(exclude_none=True)
|
||||
assert dumped["role"] == "assistant"
|
||||
assert dumped["content"] == "hello"
|
||||
assert dumped["citations"][0]["text"] == "hello"
|
||||
assert dumped["citations"][0]["sources"][0]["id"] == "d1"
|
||||
# Round-trip through the subclass to confirm the field validates.
|
||||
m2 = CohereChatMessage.model_validate(dumped)
|
||||
assert m2.citations[0].text == "hello"
|
||||
assert m2.citations[0].sources[0].id == "d1"
|
||||
@@ -1,271 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``vllm/entrypoints/cohere/protocol.py``.
|
||||
|
||||
The module is mostly a thin wrapper around the official ``cohere`` SDK
|
||||
types plus a few local additions:
|
||||
|
||||
* :class:`CohereError` envelope.
|
||||
* :class:`CohereChatV2Request` (model and messages required,
|
||||
``max_tokens`` non-negative).
|
||||
* :class:`CohereChatV2Response` plus the usage / logprob helpers.
|
||||
* The streaming event subclasses that bake a wire-format ``type``
|
||||
discriminator into ``model_dump()`` so SSE consumers can demux on it.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CitationEndEvent,
|
||||
CitationStartEvent,
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
CohereUsageTokens,
|
||||
ContentDeltaEvent,
|
||||
ContentEndEvent,
|
||||
ContentStartEvent,
|
||||
MessageEndEvent,
|
||||
MessageStartEvent,
|
||||
ToolCallDeltaEvent,
|
||||
ToolCallEndEvent,
|
||||
ToolCallStartEvent,
|
||||
ToolPlanDeltaEvent,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
# ======================================================================
|
||||
# CohereError
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereError:
|
||||
def test_model_dump_excludes_none(self):
|
||||
err = CohereError(message="boom")
|
||||
assert err.model_dump(exclude_none=True) == {"message": "boom"}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatV2Request
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatV2Request:
|
||||
def test_empty_model_rejected(self):
|
||||
with pytest.raises(VLLMValidationError, match="model is required") as exc_info:
|
||||
CohereChatV2Request(model="", messages=[{"role": "user", "content": "hi"}])
|
||||
assert exc_info.value.parameter == "model"
|
||||
assert exc_info.value.value is None
|
||||
|
||||
def test_negative_max_tokens_rejected(self):
|
||||
with pytest.raises(VLLMValidationError, match="non-negative") as exc_info:
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=-1,
|
||||
)
|
||||
assert exc_info.value.parameter == "max_tokens"
|
||||
assert exc_info.value.value == -1
|
||||
|
||||
def test_empty_messages_rejected(self):
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match="messages must contain at least one message"
|
||||
) as exc_info:
|
||||
CohereChatV2Request(model="m", messages=[])
|
||||
assert exc_info.value.parameter == "messages"
|
||||
assert exc_info.value.value is None
|
||||
|
||||
def test_zero_max_tokens_allowed(self):
|
||||
# Zero is allowed (the docs allow 0 -> return prompt only).
|
||||
req = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=0,
|
||||
)
|
||||
assert req.max_tokens == 0
|
||||
|
||||
def test_invalid_tool_choice_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice="ANY", # not REQUIRED/NONE
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("tool_fields", [{}, {"tools": None}, {"tools": []}])
|
||||
def test_required_tool_choice_requires_tools(self, tool_fields):
|
||||
with pytest.raises(VLLMValidationError, match="REQUIRED.*tools") as exc_info:
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice="REQUIRED",
|
||||
**tool_fields,
|
||||
)
|
||||
assert exc_info.value.parameter == "tool_choice"
|
||||
|
||||
@pytest.mark.parametrize("tool_choice", [None, "NONE"])
|
||||
def test_nonrequired_tool_choice_allows_empty_tools(self, tool_choice):
|
||||
request = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice=tool_choice,
|
||||
tools=[],
|
||||
)
|
||||
assert request.tool_choice == tool_choice
|
||||
|
||||
def test_stop_sequences_bounded_by_env_limit(self):
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
limit = envs.VLLM_MAX_STOP_STRINGS
|
||||
|
||||
req = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=messages,
|
||||
stop_sequences=[f"s{i}" for i in range(limit)],
|
||||
)
|
||||
assert len(req.stop_sequences) == limit
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=messages,
|
||||
stop_sequences=[f"s{i}" for i in range(limit + 1)],
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Usage / Logprobs
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestUsage:
|
||||
def test_tokens_serialization(self):
|
||||
u = CohereUsageTokens(input_tokens=10, output_tokens=5)
|
||||
assert u.model_dump() == {"input_tokens": 10.0, "output_tokens": 5.0}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatV2Response
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatV2Response:
|
||||
def test_invalid_finish_reason_rejected(self):
|
||||
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hi"}])
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Response(
|
||||
id="r1",
|
||||
finish_reason="NOT_A_REASON",
|
||||
message=msg,
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Streaming event ``type`` discriminator baked into model_dump()
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStreamingEventTypeField:
|
||||
"""Each event subclass adds a ``type: Literal[...]`` field with a
|
||||
default so ``model_dump()`` always emits the wire-format discriminator
|
||||
(the parent SDK classes don't declare ``type`` as a Pydantic field).
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls, expected_type, kwargs",
|
||||
[
|
||||
(
|
||||
MessageStartEvent,
|
||||
"message-start",
|
||||
{"id": "a", "delta": {"message": {"role": "assistant"}}},
|
||||
),
|
||||
(
|
||||
ContentStartEvent,
|
||||
"content-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"message": {"content": {"type": "text", "text": ""}}},
|
||||
},
|
||||
),
|
||||
(
|
||||
ContentDeltaEvent,
|
||||
"content-delta",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"message": {"content": {"text": "hi"}}},
|
||||
},
|
||||
),
|
||||
(ContentEndEvent, "content-end", {"index": 0}),
|
||||
(
|
||||
ToolPlanDeltaEvent,
|
||||
"tool-plan-delta",
|
||||
{"delta": {"message": {"tool_plan": "thinking"}}},
|
||||
),
|
||||
(
|
||||
ToolCallStartEvent,
|
||||
"tool-call-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {
|
||||
"tool_calls": {
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": ""},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
ToolCallDeltaEvent,
|
||||
"tool-call-delta",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {"tool_calls": {"function": {"arguments": "{}"}}}
|
||||
},
|
||||
},
|
||||
),
|
||||
(ToolCallEndEvent, "tool-call-end", {"index": 0}),
|
||||
(
|
||||
CitationStartEvent,
|
||||
"citation-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {
|
||||
"citations": {"start": 0, "end": 5, "text": "hello"}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(CitationEndEvent, "citation-end", {"index": 0}),
|
||||
(
|
||||
MessageEndEvent,
|
||||
"message-end",
|
||||
{"id": "a", "delta": {"finish_reason": "COMPLETE"}},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_type_field_default(self, cls, expected_type, kwargs):
|
||||
ev = cls(**kwargs)
|
||||
# type field is auto-populated from the Literal default.
|
||||
assert ev.type == expected_type
|
||||
# The discriminator must be present in the serialized payload so
|
||||
# clients reading the stream can demux on it.
|
||||
dumped = ev.model_dump(exclude_none=True)
|
||||
assert dumped["type"] == expected_type
|
||||
# Same in JSON form (what ``_emit`` serializes).
|
||||
assert f'"type":"{expected_type}"' in ev.model_dump_json(exclude_none=True)
|
||||
|
||||
def test_type_field_cannot_be_overridden_to_wrong_value(self):
|
||||
# Literal types reject any value other than the bake-in default.
|
||||
with pytest.raises(ValidationError):
|
||||
MessageStartEvent(
|
||||
id="a",
|
||||
delta={"message": {"role": "assistant"}},
|
||||
type="other",
|
||||
)
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sanity tests for the small registry / CLI-arg / config additions made
|
||||
by the Cohere v2 chat API:
|
||||
|
||||
* ``vllm/renderers/registry.py``: new ``"cohere"`` renderer entry.
|
||||
* ``vllm/tokenizers/registry.py``: new ``"cohere"`` tokenizer entry
|
||||
(aliased to the cached HF tokenizer).
|
||||
* ``vllm/entrypoints/openai/cli_args.py``: new
|
||||
``cohere_is_reasoning_model`` field.
|
||||
* ``vllm/config/model.py``: ``"cohere"`` added to ``TokenizerMode``
|
||||
Literal.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.model import TokenizerMode
|
||||
from vllm.entrypoints.launchers.cli_args import BaseFrontendArgs, make_arg_parser
|
||||
from vllm.renderers.registry import RENDERER_REGISTRY
|
||||
from vllm.tokenizers.registry import TokenizerRegistry
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class TestRendererRegistry:
|
||||
def test_cohere_renderer_registered(self):
|
||||
# The registry resolves to the importable ``CohereRenderer`` class.
|
||||
cls = RENDERER_REGISTRY.load_renderer_cls("cohere")
|
||||
assert cls.__name__ == "CohereRenderer"
|
||||
# Sanity: class lives in the cohere renderer module.
|
||||
assert cls.__module__ == "vllm.renderers.cohere"
|
||||
|
||||
|
||||
class TestTokenizerRegistry:
|
||||
def test_cohere_aliased_to_cached_hf_tokenizer(self):
|
||||
# ``cohere`` mode uses the standard HF tokenizer; only the
|
||||
# renderer stage is replaced. This test guards against accidental
|
||||
# divergence.
|
||||
cls = TokenizerRegistry.load_tokenizer_cls("cohere")
|
||||
assert cls.__name__ == "CachedHfTokenizer"
|
||||
|
||||
|
||||
class TestTokenizerModeLiteral:
|
||||
def test_cohere_is_a_valid_tokenizer_mode(self):
|
||||
# The Literal must enumerate ``"cohere"`` so engine arg parsing
|
||||
# accepts ``--tokenizer-mode cohere``.
|
||||
modes = typing.get_args(TokenizerMode)
|
||||
assert "cohere" in modes
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# ``--cohere-is-reasoning-model`` CLI flag
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCohereCliArg:
|
||||
"""Verifies the new ``--cohere-is-reasoning-model`` flag end-to-end
|
||||
through ``make_arg_parser`` — mirrors the pattern used in
|
||||
:mod:`tests.entrypoints.openai.test_cli_args`.
|
||||
"""
|
||||
|
||||
def test_default_value_on_dataclass_is_true(self):
|
||||
fields = {f.name: f for f in dataclasses.fields(BaseFrontendArgs)}
|
||||
assert "cohere_is_reasoning_model" in fields
|
||||
field = fields["cohere_is_reasoning_model"]
|
||||
assert field.default is True
|
||||
assert field.type is bool
|
||||
|
||||
@pytest.fixture
|
||||
def serve_parser(self) -> FlexibleArgumentParser:
|
||||
parser = FlexibleArgumentParser()
|
||||
return make_arg_parser(parser)
|
||||
|
||||
def test_default_via_argparse_is_true(self, serve_parser: FlexibleArgumentParser):
|
||||
# No flag supplied → dataclass default (True) wins.
|
||||
args = serve_parser.parse_args(["--model", "m"])
|
||||
assert args.cohere_is_reasoning_model is True
|
||||
|
||||
def test_explicit_false_via_argparse(self, serve_parser: FlexibleArgumentParser):
|
||||
# Boolean dataclass fields are wired up as ``--flag value`` /
|
||||
# ``--no-flag`` pairs by FlexibleArgumentParser.
|
||||
args = serve_parser.parse_args(
|
||||
["--model", "m", "--no-cohere-is-reasoning-model"]
|
||||
)
|
||||
assert args.cohere_is_reasoning_model is False
|
||||
|
||||
def test_explicit_true_via_argparse(self, serve_parser: FlexibleArgumentParser):
|
||||
args = serve_parser.parse_args(["--model", "m", "--cohere-is-reasoning-model"])
|
||||
assert args.cohere_is_reasoning_model is True
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,952 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Cohere v2 SSE stream conversion in
|
||||
``vllm/entrypoints/cohere/serving.py``.
|
||||
|
||||
The stream-translation entry point is
|
||||
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
|
||||
async iterable of OpenAI chunk objects into Cohere's
|
||||
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
|
||||
event stream.
|
||||
|
||||
We test the helpers (``_StreamState``, ``_handle_*_delta``, etc.) in
|
||||
isolation, plus a handful of end-to-end scenarios that exercise the
|
||||
state machine.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation as VLLMCitation,
|
||||
)
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
CitationSource,
|
||||
)
|
||||
from vllm.entrypoints.cohere.protocol import MessageStartEvent
|
||||
from vllm.entrypoints.cohere.serving import (
|
||||
_DONE_FRAME,
|
||||
CohereServingChatV2,
|
||||
ContentBlockType,
|
||||
_emit,
|
||||
_sse,
|
||||
_StreamState,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeServing(CohereServingChatV2):
|
||||
"""Lightweight stand-in for :class:`CohereServingChatV2` that skips
|
||||
the heavy ``OpenAIServingChat.__init__`` chain (which would need a
|
||||
real engine client, model registry, render service, etc.).
|
||||
|
||||
Only ``_is_reasoning_model`` is read by the methods under test
|
||||
(``_chat_completion_stream_to_v2`` and the per-delta handlers); the
|
||||
rest is dead weight for unit testing.
|
||||
"""
|
||||
|
||||
def __init__(self, is_reasoning_model: bool = True) -> None:
|
||||
# Intentionally skipping super().__init__ — see class docstring.
|
||||
self._is_reasoning_model = is_reasoning_model
|
||||
|
||||
|
||||
def _serving(is_reasoning_model: bool = True) -> CohereServingChatV2:
|
||||
return _FakeServing(is_reasoning_model=is_reasoning_model)
|
||||
|
||||
|
||||
def _parse_event(frame: str) -> dict[str, Any]:
|
||||
"""Strip the ``data: ... \\n\\n`` wrapper and parse the JSON payload."""
|
||||
assert frame.startswith("data: ")
|
||||
assert frame.endswith("\n\n")
|
||||
return json.loads(frame[len("data: ") : -2])
|
||||
|
||||
|
||||
def _make_chunk(
|
||||
*,
|
||||
chunk_id: str = "chunk_0",
|
||||
role: str | None = None,
|
||||
content: str | None = None,
|
||||
reasoning: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
finish_reason: str | None = None,
|
||||
stop_reason: int | str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
omit_choices: bool = False,
|
||||
citations: list[Any] | None = None,
|
||||
) -> ChatCompletionStreamResponse:
|
||||
"""Build the raw chat chunk the production converter consumes."""
|
||||
delta: dict[str, Any] = {}
|
||||
if role is not None:
|
||||
delta["role"] = role
|
||||
if content is not None:
|
||||
delta["content"] = content
|
||||
if reasoning is not None:
|
||||
delta["reasoning"] = reasoning
|
||||
if tool_calls is not None:
|
||||
delta["tool_calls"] = tool_calls
|
||||
if citations is not None:
|
||||
delta["citations"] = citations
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
}
|
||||
if not omit_choices:
|
||||
payload["choices"] = [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": delta,
|
||||
"finish_reason": finish_reason,
|
||||
"stop_reason": stop_reason,
|
||||
}
|
||||
]
|
||||
else:
|
||||
payload["choices"] = []
|
||||
if usage is not None:
|
||||
payload["usage"] = usage
|
||||
return ChatCompletionStreamResponse.model_validate(payload)
|
||||
|
||||
|
||||
async def _stream_from(
|
||||
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
|
||||
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
|
||||
async def _drain(
|
||||
serving: CohereServingChatV2,
|
||||
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
|
||||
) -> list[str]:
|
||||
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
||||
the emitted SSE frames."""
|
||||
gen = serving._chat_completion_stream_to_v2(_stream_from(items))
|
||||
return [frame async for frame in gen]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Low-level helpers: _sse, _emit, _DONE_FRAME
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestSSEHelpers:
|
||||
def test_sse_wraps_payload(self):
|
||||
assert _sse("hello") == "data: hello\n\n"
|
||||
|
||||
def test_done_frame_constant(self):
|
||||
# cohere-python and Fern-generated clients key off this exact
|
||||
# sentinel; keep it byte-for-byte stable.
|
||||
assert _DONE_FRAME == "data: [DONE]\n\n"
|
||||
|
||||
def test_emit_serializes_event_with_type_discriminator(self):
|
||||
frame = _emit(
|
||||
MessageStartEvent(id="abc", delta={"message": {"role": "assistant"}})
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["type"] == "message-start"
|
||||
assert payload["id"] == "abc"
|
||||
assert payload["delta"] == {"message": {"role": "assistant"}}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _StreamState
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStreamState:
|
||||
def test_content_index_monotonic(self):
|
||||
st = _StreamState()
|
||||
assert st.next_content_index() == 0
|
||||
assert st.next_content_index() == 1
|
||||
assert st.next_content_index() == 2
|
||||
|
||||
def test_citation_index_separate_from_content_index(self):
|
||||
st = _StreamState()
|
||||
st.next_content_index() # 0
|
||||
st.next_content_index() # 1
|
||||
# Citation indexing is independent of content indexing.
|
||||
assert st.next_citation_index() == 0
|
||||
assert st.next_citation_index() == 1
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _close_open_blocks
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCloseOpenBlocks:
|
||||
def test_no_block_open_emits_nothing(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
assert serving._close_open_blocks(state) == []
|
||||
|
||||
def test_text_block_emits_content_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.TEXT
|
||||
state.active_block_index = 1
|
||||
out = serving._close_open_blocks(state)
|
||||
assert len(out) == 1
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "content-end", "index": 1}
|
||||
assert state.active_block is None
|
||||
assert state.active_block_index is None
|
||||
|
||||
def test_thinking_block_emits_content_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.THINKING
|
||||
state.active_block_index = 3
|
||||
out = serving._close_open_blocks(state)
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "content-end", "index": 3}
|
||||
|
||||
def test_tool_call_block_emits_tool_call_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.TOOL_CALL
|
||||
state.active_tool_index = 7
|
||||
out = serving._close_open_blocks(state)
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "tool-call-end", "index": 7}
|
||||
assert state.active_tool_index is None
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_text_delta
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleTextDelta:
|
||||
def test_opens_block_first_time(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
events = serving._handle_text_delta(state, "Hi")
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
delta = _parse_event(events[1])
|
||||
assert start["type"] == "content-start"
|
||||
assert start["index"] == 0
|
||||
assert start["delta"]["message"]["content"]["type"] == "text"
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["index"] == 0
|
||||
assert delta["delta"]["message"]["content"]["text"] == "Hi"
|
||||
assert state.active_block == ContentBlockType.TEXT
|
||||
assert state.active_block_index == 0
|
||||
|
||||
def test_continues_block_with_just_delta(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
serving._handle_text_delta(state, "Hi")
|
||||
events = serving._handle_text_delta(state, " there")
|
||||
# Only a delta event, no new content-start.
|
||||
assert len(events) == 1
|
||||
delta = _parse_event(events[0])
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["delta"]["message"]["content"]["text"] == " there"
|
||||
|
||||
def test_switches_from_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
# Open a thinking block first, then switch to text.
|
||||
serving._handle_thinking_delta(state, "ponder")
|
||||
events = serving._handle_text_delta(state, "answer")
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "content-start", "content-delta"]
|
||||
# The text block gets a new index (1), distinct from thinking's 0.
|
||||
assert _parse_event(events[1])["index"] == 1
|
||||
assert state.active_block == ContentBlockType.TEXT
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_thinking_delta
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleThinkingDelta:
|
||||
def test_reasoning_model_opens_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
events = serving._handle_thinking_delta(state, "thought")
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
delta = _parse_event(events[1])
|
||||
assert start["type"] == "content-start"
|
||||
assert start["delta"]["message"]["content"]["type"] == "thinking"
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["delta"]["message"]["content"]["thinking"] == "thought"
|
||||
assert state.active_block == ContentBlockType.THINKING
|
||||
|
||||
def test_reasoning_model_continues_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
serving._handle_thinking_delta(state, "first")
|
||||
events = serving._handle_thinking_delta(state, " more")
|
||||
assert len(events) == 1
|
||||
assert _parse_event(events[0])["type"] == "content-delta"
|
||||
|
||||
def test_non_reasoning_model_emits_tool_plan_delta(self):
|
||||
# Older Command models stream reasoning as ``tool_plan`` deltas;
|
||||
# no content-start/end pair is emitted.
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
events = serving._handle_thinking_delta(state, "planning")
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-plan-delta"
|
||||
assert payload["delta"]["message"]["tool_plan"] == "planning"
|
||||
# ``tool_plan`` deltas don't claim an active content block.
|
||||
assert state.active_block is None
|
||||
|
||||
def test_non_reasoning_model_closes_open_text_block(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
# Open a text block first.
|
||||
serving._handle_text_delta(state, "answer")
|
||||
events = serving._handle_thinking_delta(state, "rethink")
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "tool-plan-delta"]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_tool_call_deltas
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleToolCallDeltas:
|
||||
def test_new_tool_call_opens_tool_call_start(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
deltas = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type(
|
||||
"Fn", (), {"name": "calc", "arguments": '{"x":'}
|
||||
)(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, deltas)
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-call-start"
|
||||
assert payload["index"] == 0
|
||||
tc = payload["delta"]["message"]["tool_calls"]
|
||||
assert tc["id"] == "c1"
|
||||
assert tc["function"]["name"] == "calc"
|
||||
assert tc["function"]["arguments"] == '{"x":'
|
||||
assert state.active_block == ContentBlockType.TOOL_CALL
|
||||
assert state.active_tool_index == 0
|
||||
assert 0 in state.tool_calls_seen
|
||||
|
||||
def test_subsequent_arguments_emit_delta(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
# First call: start.
|
||||
first = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type("Fn", (), {"name": "calc", "arguments": ""})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
serving._handle_tool_call_deltas(state, first)
|
||||
# Second call: same index, additional arguments fragment.
|
||||
more = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": None,
|
||||
"function": type("Fn", (), {"name": None, "arguments": "1}"})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, more)
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-call-delta"
|
||||
assert payload["index"] == 0
|
||||
assert (
|
||||
payload["delta"]["message"]["tool_calls"]["function"]["arguments"] == "1}"
|
||||
)
|
||||
|
||||
def test_new_tool_call_closes_existing_content_block(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
# Open a text block, then start a tool call.
|
||||
serving._handle_text_delta(state, "I'll call:")
|
||||
deltas = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type("Fn", (), {"name": "calc", "arguments": "{}"})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, deltas)
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "tool-call-start"]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_citation_deltas
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleCitationDeltas:
|
||||
"""Test the per-delta emitter. Sources are pre-resolved by the
|
||||
reasoning parser via the position map forwarded through
|
||||
``chat_template_kwargs`` (see :func:`_melody_sources_to_vllm` in
|
||||
``vllm/reasoning/cohere_command_reasoning_parser.py``), so the
|
||||
handler only has to coerce to the SDK wire shape, apply the
|
||||
``THINKING_CONTENT`` -> ``PLAN`` rewrite, and drop citations with
|
||||
no attributable sources.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _resolved_source(**overrides: Any) -> CitationSource:
|
||||
base = dict(
|
||||
type="document",
|
||||
id="doc_hamlet",
|
||||
document={"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
|
||||
)
|
||||
base.update(overrides)
|
||||
return CitationSource(**base)
|
||||
|
||||
def test_resolved_citation_emits_start_and_end_with_real_id(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=11,
|
||||
text="Shakespeare",
|
||||
sources=[self._resolved_source()],
|
||||
type="TEXT_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
end = _parse_event(events[1])
|
||||
assert start["type"] == "citation-start"
|
||||
assert start["index"] == 0
|
||||
cit_payload = start["delta"]["message"]["citations"]
|
||||
assert cit_payload["start"] == 0
|
||||
assert cit_payload["end"] == 11
|
||||
assert cit_payload["text"] == "Shakespeare"
|
||||
assert cit_payload["sources"] == [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "doc_hamlet",
|
||||
"document": {"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
|
||||
}
|
||||
]
|
||||
assert end == {"type": "citation-end", "index": 0}
|
||||
|
||||
def test_multiple_sources_survive_to_wire(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="works",
|
||||
sources=[
|
||||
self._resolved_source(id="d0", document={"id": "d0"}),
|
||||
self._resolved_source(id="d1", document={"id": "d1"}),
|
||||
],
|
||||
type="TEXT_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
ids = [s["id"] for s in cit_payload["sources"]]
|
||||
assert ids == ["d0", "d1"]
|
||||
|
||||
def test_source_without_id_streamed_with_payload(self):
|
||||
# Match the cohere api: id-less tool sources (from text-only
|
||||
# tool results) still ride through with their ``tool_output``
|
||||
# payload; only the ``id`` field is omitted.
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[CitationSource(type="tool", tool_output={"content": "hi"})],
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert len(events) >= 1
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
assert cit_payload["sources"] == [
|
||||
{"type": "tool", "tool_output": {"content": "hi"}}
|
||||
]
|
||||
assert "id" not in cit_payload["sources"][0]
|
||||
|
||||
def test_citation_with_no_sources_streamed_dropped(self):
|
||||
# Citations whose source list is entirely empty are still
|
||||
# dropped so we don't emit an unattributed anchor.
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[],
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert events == []
|
||||
|
||||
def test_plan_rewrite_on_non_reasoning_model(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[self._resolved_source()],
|
||||
type="THINKING_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
assert cit_payload["type"] == "PLAN"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _build_message_end_event
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestBuildMessageEndEvent:
|
||||
def test_without_usage(self):
|
||||
serving = _serving()
|
||||
frame = serving._build_message_end_event(chunk_id="abc", finish_reason="stop")
|
||||
payload = _parse_event(frame)
|
||||
assert payload["type"] == "message-end"
|
||||
assert payload["id"] == "abc"
|
||||
assert payload["delta"]["finish_reason"] == "COMPLETE"
|
||||
assert "usage" not in payload["delta"]
|
||||
|
||||
def test_with_usage(self):
|
||||
serving = _serving()
|
||||
chunk = ChatCompletionStreamResponse.model_validate(
|
||||
{
|
||||
"id": "abc",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
},
|
||||
}
|
||||
)
|
||||
frame = serving._build_message_end_event(
|
||||
chunk_id="abc", finish_reason="length", usage_chunk=chunk
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["delta"]["finish_reason"] == "MAX_TOKENS"
|
||||
usage = payload["delta"]["usage"]
|
||||
assert usage["billed_units"] == {"input_tokens": 10, "output_tokens": 5}
|
||||
assert usage["tokens"] == {"input_tokens": 10, "output_tokens": 5}
|
||||
assert "cached_tokens" not in usage
|
||||
|
||||
def test_with_cached_tokens(self):
|
||||
serving = _serving()
|
||||
chunk = ChatCompletionStreamResponse.model_validate(
|
||||
{
|
||||
"id": "abc",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
"prompt_tokens_details": {"cached_tokens": 3},
|
||||
},
|
||||
}
|
||||
)
|
||||
frame = serving._build_message_end_event(
|
||||
chunk_id="abc", finish_reason="stop", usage_chunk=chunk
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["delta"]["usage"]["cached_tokens"] == 3
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# End-to-end: _chat_completion_stream_to_v2
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestChatCompletionStreamToV2:
|
||||
"""End-to-end stream lifecycle tests."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_only_happy_path_emits_full_lifecycle(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="Hi"),
|
||||
_make_chunk(content=" there"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 4,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 6,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# message-start, content-start, content-delta, content-delta,
|
||||
# content-end, message-end, then [DONE] as the last frame.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# message-end should carry usage stats from the trailing chunk.
|
||||
end_payload = _parse_event(frames[-2])
|
||||
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
|
||||
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_sequence_finish_reason(self):
|
||||
serving = _serving()
|
||||
frames = await _drain(
|
||||
serving,
|
||||
[
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(
|
||||
finish_reason="stop",
|
||||
stop_reason="<END>",
|
||||
),
|
||||
_make_chunk(omit_choices=True),
|
||||
],
|
||||
)
|
||||
end_payload = _parse_event(frames[-2])
|
||||
assert end_payload["delta"]["finish_reason"] == "STOP_SEQUENCE"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_then_tool_call_closes_text_first(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="planning..."),
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "calc", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="tool_calls"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# Text block is opened+delta, then closed before the tool-call
|
||||
# opens, and the final close happens on the usage chunk path.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"tool-call-start",
|
||||
"tool-call-end",
|
||||
"message-end",
|
||||
]
|
||||
# finish_reason captured from the prior chunk.
|
||||
end = _parse_event(frames[-2])
|
||||
assert end["delta"]["finish_reason"] == "TOOL_CALL"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thinking_then_text_reasoning_model(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(reasoning="thinking..."),
|
||||
_make_chunk(content="answer"),
|
||||
_make_chunk(finish_reason="stop"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# Thinking block opens with index 0; text block reopens with index 1.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
thinking_start = _parse_event(frames[1])
|
||||
assert thinking_start["delta"]["message"]["content"]["type"] == "thinking"
|
||||
text_start = _parse_event(frames[4])
|
||||
assert text_start["delta"]["message"]["content"]["type"] == "text"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_on_non_reasoning_model_emits_tool_plan_delta(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(reasoning="planning"),
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="tool_calls"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types == [
|
||||
"message-start",
|
||||
"tool-plan-delta",
|
||||
"tool-call-start",
|
||||
"tool-call-end",
|
||||
"message-end",
|
||||
]
|
||||
# No thinking content blocks should be present.
|
||||
assert "content-start" not in types
|
||||
assert "content-end" not in types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_exhaustion_closes_open_block_without_usage(self):
|
||||
# The raw stream may finish without a trailing usage-only chunk.
|
||||
# The translator must still emit message-end before [DONE] so
|
||||
# Cohere clients don't hang.
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="Hi"),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_call"])
|
||||
async def test_upstream_error_preserves_message(self, block_type):
|
||||
serving = _serving()
|
||||
items: list[ChatCompletionStreamResponse | ErrorResponse] = []
|
||||
if block_type is not None:
|
||||
items.append(_make_chunk(role="assistant"))
|
||||
if block_type == "text":
|
||||
items.append(_make_chunk(content="Hi"))
|
||||
elif block_type == "thinking":
|
||||
items.append(_make_chunk(reasoning="Thinking"))
|
||||
else:
|
||||
items.append(
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"function": {"name": "search", "arguments": '{"q":'},
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
items.append(serving.create_error_response(ValueError("backend failed")))
|
||||
|
||||
frames = await _drain(serving, items)
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
events = [_parse_event(frame) for frame in frames[:-1]]
|
||||
error_ends = [event for event in events if event["type"] == "message-end"]
|
||||
assert len(error_ends) == 1
|
||||
assert error_ends[0]["delta"] == {
|
||||
"finish_reason": "ERROR",
|
||||
"error": "backend failed",
|
||||
}
|
||||
assert events[-1] == error_ends[0]
|
||||
if block_type is not None:
|
||||
assert events[-2]["type"] == (
|
||||
"tool-call-end" if block_type == "tool_call" else "content-end"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("started", [False, True])
|
||||
async def test_error_message_end_is_sanitized(self, started):
|
||||
# The client-visible ``error`` field on the terminal message-end
|
||||
# must be routed through ``sanitize_message`` so tracebacks,
|
||||
# host filesystem paths, and Python object memory addresses
|
||||
# can't leak from a mid-stream exception into the SSE payload.
|
||||
serving = _serving()
|
||||
|
||||
async def _raises_mid_stream() -> AsyncGenerator[
|
||||
ChatCompletionStreamResponse | ErrorResponse, None
|
||||
]:
|
||||
if started:
|
||||
yield _make_chunk(role="assistant")
|
||||
raise RuntimeError(
|
||||
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
|
||||
)
|
||||
|
||||
frames = [
|
||||
f async for f in serving._chat_completion_stream_to_v2(_raises_mid_stream())
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
error_end = _parse_event(frames[-2])
|
||||
assert error_end["delta"]["finish_reason"] == "ERROR"
|
||||
err_msg = error_end["delta"]["error"]
|
||||
# Memory address stripped and path replaced with ``<path>``.
|
||||
assert "0x7fabc0deface" not in err_msg
|
||||
assert "/Users/dev/vllm/x.py" not in err_msg
|
||||
assert "<path>" in err_msg
|
||||
# The human-facing prefix should still survive sanitization.
|
||||
assert "boom" in err_msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_citations_in_delta_emit_citation_events(self):
|
||||
# End-to-end streaming citation flow. The parser hands us
|
||||
# already-resolved sources (see ``_melody_sources_to_vllm``
|
||||
# in ``vllm/reasoning/cohere_command_reasoning_parser.py``);
|
||||
# the streaming loop just has to coerce and emit.
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="hello"),
|
||||
_make_chunk(
|
||||
citations=[
|
||||
{
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "hello",
|
||||
"sources": [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
"document": {"id": "d1", "text": "cited"},
|
||||
}
|
||||
],
|
||||
"type": "TEXT_CONTENT",
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="stop"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
parsed = [_parse_event(f) for f in frames[:-1]]
|
||||
types = [p["type"] for p in parsed]
|
||||
assert "citation-start" in types
|
||||
assert "citation-end" in types
|
||||
cit_start = next(p for p in parsed if p["type"] == "citation-start")
|
||||
assert cit_start["delta"]["message"]["citations"]["sources"] == [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
"document": {"id": "d1", "text": "cited"},
|
||||
}
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_chunk_emits_message_start_with_chunk_id(self):
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(chunk_id="my-id", role="assistant"),
|
||||
_make_chunk(chunk_id="my-id", content="hi"),
|
||||
_make_chunk(
|
||||
chunk_id="my-id",
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
ms = _parse_event(frames[0])
|
||||
assert ms["type"] == "message-start"
|
||||
assert ms["id"] == "my-id"
|
||||
assert ms["delta"]["message"]["role"] == "assistant"
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.openai.decisions.adapters import (
|
||||
input_text,
|
||||
make_answer,
|
||||
make_read_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.protocol import DecisionRequest
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
StructuredDecisionError,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def request(**kwargs):
|
||||
return DecisionRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "The package is broken.",
|
||||
"questions": [{"type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_predicate_returns_probability_of_true_and_null_name():
|
||||
question = request().questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["false", "true"]
|
||||
assert make_answer(question, [0.1, 0.9], 0.5).model_dump() == {
|
||||
"type": "predicate",
|
||||
"name": None,
|
||||
"probability": 0.9,
|
||||
}
|
||||
|
||||
|
||||
def test_choice_preserves_boolean_and_string_values():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "decision",
|
||||
"instructions": "Select a value.",
|
||||
"choices": [{"value": True}, {"value": "true"}, {"value": False}],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["true", '"true"', "false"]
|
||||
answer = make_answer(question, [0.7, 0.2, 0.1], 0.5).model_dump()
|
||||
assert answer["name"] == "decision"
|
||||
assert answer["choice"] is True
|
||||
assert answer["probabilities"] == [
|
||||
{"value": True, "probability": 0.7},
|
||||
{"value": "true", "probability": 0.2},
|
||||
{"value": False, "probability": 0.1},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.35)
|
||||
|
||||
|
||||
def test_score_is_weighted_average_of_zero_based_level_indices():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [{"label": label} for label in ("low", "medium", "high")],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
answer = make_answer(question, [0.1, 0.7, 0.2], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(1.1)
|
||||
assert answer["probabilities"] == [
|
||||
{"value": 0, "label": "low", "probability": 0.1},
|
||||
{"value": 1, "label": "medium", "probability": 0.7},
|
||||
{"value": 2, "label": "high", "probability": 0.2},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.56)
|
||||
|
||||
|
||||
def test_text_messages_preserve_evidence_order():
|
||||
parsed = request(
|
||||
input=[
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "second"},
|
||||
{"type": "input_text", "text": "third"},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
assert input_text(parsed.input) == "first\n\nsecond\nthird"
|
||||
assert input_text([]) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("evidence", ["", [], [{"role": "user", "content": []}]])
|
||||
def test_empty_input_is_valid_under_the_openai_schema(evidence):
|
||||
assert input_text(request(input=evidence).input) == ""
|
||||
|
||||
|
||||
def test_score_levels_with_the_same_label_keep_their_positions():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [
|
||||
{"label": "damage", "description": "cosmetic"},
|
||||
{"label": "damage", "description": "unusable"},
|
||||
],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert read.labels == ("A", "B")
|
||||
answer = make_answer(question, [0.1, 0.9], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(0.9)
|
||||
assert [p["value"] for p in answer["probabilities"]] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override,match",
|
||||
[
|
||||
({"input": [{"role": "assistant", "content": "x"}]}, "user"),
|
||||
(
|
||||
{
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,AA==",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"text input only",
|
||||
),
|
||||
({"input": {"text": "x"}}, "Input should"),
|
||||
({"questions": []}, "at least 1"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "name": None}]},
|
||||
"name must be a string",
|
||||
),
|
||||
({"questions": [{"type": "predicate", "instructions": 1}]}, "valid string"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "criteria": {}}]},
|
||||
"Extra inputs",
|
||||
),
|
||||
({"safety_identifier": "x" * 129}, "at most 128"),
|
||||
({"stream": True}, "Extra inputs"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_or_unsupported_inputs(override, match):
|
||||
with pytest.raises(ValidationError, match=match):
|
||||
request(**override)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [[True, True], ["a", "a"], [1, "a"], [None, "a"]])
|
||||
def test_choice_rejects_duplicates_and_non_string_non_boolean_values(values):
|
||||
with pytest.raises(ValidationError):
|
||||
request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": v} for v in values],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (255, True), (256, False)]
|
||||
)
|
||||
def test_choice_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
assert len(request(**data).questions[0].choices) == count
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (10, True), (11, False)]
|
||||
)
|
||||
def test_score_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "x",
|
||||
"levels": [{"label": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
request(**data)
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
def test_question_limit_and_optional_safety_identifier():
|
||||
questions = [{"type": "predicate", "instructions": "x"}] * 200
|
||||
assert len(request(questions=questions, safety_identifier=None).questions) == 200
|
||||
with pytest.raises(ValidationError, match="at most 200"):
|
||||
request(questions=[*questions, questions[0]])
|
||||
|
||||
|
||||
def test_backend_rejects_more_choices_than_single_token_labels():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(27)],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 choices"):
|
||||
make_read_question(0, question)
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
QUESTION_TYPES,
|
||||
Option,
|
||||
Question,
|
||||
QuestionType,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
register_question_type,
|
||||
)
|
||||
|
||||
|
||||
def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"choice",
|
||||
instructions,
|
||||
criteria or {"billing": "money", "outage": None, "other": None},
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
|
||||
return build_question(qid, "noul", instructions, criteria, 128)
|
||||
|
||||
|
||||
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"score",
|
||||
instructions,
|
||||
criteria or ["calm", "annoyed", "furious"],
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def test_choice_labels_follow_option_order():
|
||||
q = choice()
|
||||
assert q.labels == ("A", "B", "C")
|
||||
assert [a.name for a in q.options] == ["billing", "outage", "other"]
|
||||
assert q.options[0].description == "money"
|
||||
wide = choice(criteria={str(i): None for i in range(26)})
|
||||
assert wide.labels[-1] == "Z"
|
||||
|
||||
|
||||
def test_choice_prompt():
|
||||
q = choice()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Which team?\n"
|
||||
"A: billing - money\n"
|
||||
"B: outage\n"
|
||||
"C: other\n"
|
||||
"Answer with the letter of one option only."
|
||||
)
|
||||
|
||||
|
||||
def test_choice_answer_shape():
|
||||
q = choice()
|
||||
answer = q.type.answer(q, [0.2, 0.7, 0.1], 0.5)
|
||||
assert answer == {
|
||||
"type": "choice",
|
||||
"choice": "outage",
|
||||
"probabilities": {"billing": 0.2, "outage": 0.7, "other": 0.1},
|
||||
"confidence": 0.35,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qid,type_name,criteria,match",
|
||||
[
|
||||
("", "choice", {"x": None, "y": None}, "non-empty"),
|
||||
("q", "nope", {"x": None, "y": None}, "unknown question type"),
|
||||
("q", "choice", ["x", "y"], "must map option names"),
|
||||
("q", "noul", ["yes", "no"], "must be an object"),
|
||||
("q", "noul", {"maybe": "same day"}, "must be an object"),
|
||||
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
|
||||
("q", "score", ["calm"], "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_build_question_rejects(qid, type_name, criteria, match):
|
||||
with pytest.raises(StructuredDecisionError, match=match):
|
||||
build_question(qid, type_name, "", criteria, 128)
|
||||
|
||||
|
||||
def test_one_option_choice():
|
||||
q = choice(criteria={"only": None})
|
||||
assert q.labels == ("A",)
|
||||
assert q.type.answer(q, [1.0], 0.4)["confidence"] == 0.4
|
||||
|
||||
|
||||
def test_option_limit():
|
||||
with pytest.raises(StructuredDecisionError, match="at most 2 options"):
|
||||
choice(criteria={"x": None, "y": None, "z": None}, max_options=2)
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 options"):
|
||||
choice(criteria={str(i): None for i in range(27)})
|
||||
|
||||
|
||||
def test_noul_labels_and_prompt():
|
||||
q = noul()
|
||||
assert q.labels == ("yes", "no")
|
||||
assert [a.name for a in q.options] == ["yes", "no"]
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Reply within the hour?\nyes\nno\nAnswer with yes or no only."
|
||||
)
|
||||
described = noul(criteria={"true": "same day", "false": "later"})
|
||||
assert described.type.prompt(described) == (
|
||||
"Question: Reply within the hour?\n"
|
||||
"yes: same day\n"
|
||||
"no: later\n"
|
||||
"Answer with yes or no only."
|
||||
)
|
||||
|
||||
|
||||
def test_noul_answer_shape():
|
||||
q = noul()
|
||||
answer = q.type.answer(q, [0.8, 0.2], 0.5)
|
||||
assert answer == {
|
||||
"type": "noul",
|
||||
"noul": 0.8,
|
||||
"probabilities": {"yes": 0.8, "no": 0.2},
|
||||
"confidence": 0.4,
|
||||
}
|
||||
|
||||
|
||||
def test_score_labels_are_levels():
|
||||
assert score().labels == ("0", "1", "2")
|
||||
with pytest.raises(StructuredDecisionError, match="at most 10"):
|
||||
score(criteria=[str(i) for i in range(11)])
|
||||
|
||||
|
||||
def test_score_prompt():
|
||||
q = score()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: How angry?\n"
|
||||
"0: calm\n"
|
||||
"1: annoyed\n"
|
||||
"2: furious\n"
|
||||
"Answer with the number of one level only."
|
||||
)
|
||||
|
||||
|
||||
def test_score_answer_shape():
|
||||
q = score()
|
||||
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
|
||||
assert answer == {
|
||||
"type": "score",
|
||||
"score": 0.75,
|
||||
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
|
||||
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
|
||||
"confidence": 0.2,
|
||||
}
|
||||
|
||||
|
||||
def test_registered_type_plugs_in():
|
||||
class BinaryQuestion(QuestionType):
|
||||
name = "test_binary"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
return [Option("yes"), Option("no")]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {"type": self.name, "yes": probs[0]}
|
||||
|
||||
register_question_type(BinaryQuestion)
|
||||
try:
|
||||
q = build_question("ok", "test_binary", "Is it fine?", None, 128)
|
||||
assert q.labels == ("A", "B")
|
||||
assert q.type.answer(q, [0.9, 0.1], 1.0) == {"type": "test_binary", "yes": 0.9}
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
register_question_type(BinaryQuestion)
|
||||
finally:
|
||||
del QUESTION_TYPES["test_binary"]
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def post(server, body):
|
||||
return requests.post(server.url_for("v1/systemone"), json=body)
|
||||
|
||||
|
||||
def choice(instructions, *names):
|
||||
return {
|
||||
"type": "choice",
|
||||
"instructions": instructions,
|
||||
"criteria": dict.fromkeys(names),
|
||||
}
|
||||
|
||||
|
||||
COPY = {
|
||||
"team": choice(
|
||||
"Which team does the message name?", "billing", "shipping", "security"
|
||||
),
|
||||
"lang": choice("Which language does the message name?", "English", "French"),
|
||||
}
|
||||
|
||||
|
||||
def test_choice_decision(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"team": "billing", "language": "French"},
|
||||
"questions": COPY,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["object"] == "structured_decision"
|
||||
assert list(body["answers"]) == ["team", "lang"]
|
||||
team = body["answers"]["team"]
|
||||
assert team["type"] == "choice"
|
||||
assert set(team["probabilities"]) == {"billing", "shipping", "security"}
|
||||
assert sum(team["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert team["confidence"] == pytest.approx(
|
||||
max(team["probabilities"].values()) * body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
for diag in body["diagnostics"].values():
|
||||
assert 0.0 < diag["label_mass"] <= 1.0 + 1e-6
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
assert body["usage"]["input_tokens"] > 0
|
||||
|
||||
|
||||
def test_one_option_choice(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "Team: billing.",
|
||||
"questions": {
|
||||
"team": choice("Which team does the message name?", "billing")
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
team = body["answers"]["team"]
|
||||
assert team["choice"] == "billing"
|
||||
assert team["probabilities"] == {"billing": 1.0}
|
||||
assert team["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
def test_one_option_coin_flip(server):
|
||||
# A fair coin with only "heads" allowed. The answer is forced, so its
|
||||
# confidence must come from the model's probability of the label.
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "I flip a fair coin.",
|
||||
"questions": {"coin": choice("How did the coin land?", "heads")},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
coin = body["answers"]["coin"]
|
||||
assert coin["probabilities"] == {"heads": 1.0}
|
||||
assert coin["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["coin"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state,questions,expected",
|
||||
[
|
||||
(
|
||||
{"team": "billing", "language": "French"},
|
||||
COPY,
|
||||
{"team": "billing", "lang": "French"},
|
||||
),
|
||||
(
|
||||
"My card was charged twice for one order.",
|
||||
{
|
||||
"card": choice("Does the message mention a card?", "yes", "no"),
|
||||
"mood": choice("How does the customer feel?", "happy", "angry"),
|
||||
},
|
||||
{"card": "yes", "mood": "angry"},
|
||||
),
|
||||
(
|
||||
"Hello.",
|
||||
{
|
||||
"sky": choice(
|
||||
"What color is the sky on a clear day?", "blue", "green", "red"
|
||||
)
|
||||
},
|
||||
{"sky": "blue"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_single_read_answers(server, state, questions, expected):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": state,
|
||||
"questions": questions,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
answers = response.json()["answers"]
|
||||
assert {qid: a["choice"] for qid, a in answers.items()} == expected
|
||||
|
||||
|
||||
def test_option_limit(server):
|
||||
body = {
|
||||
"model": MODEL_NAME,
|
||||
"state": "Pick option 7.",
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
ok = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(26)))}}
|
||||
)
|
||||
assert ok.status_code == 200, ok.text
|
||||
assert len(ok.json()["answers"]["n"]["probabilities"]) == 26
|
||||
over = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(27)))}}
|
||||
)
|
||||
assert over.status_code == 400
|
||||
assert "at most 26 options" in over.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_thinking_is_refused(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "x",
|
||||
"questions": {"q": choice("Which?", "a", "b")},
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "thinking must be off" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_noul_and_score(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"ticket": "My card was charged twice for one order."},
|
||||
"questions": {
|
||||
"urgent": {"type": "noul", "instructions": "Is money moving?"},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset is the customer?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
urgent = body["answers"]["urgent"]
|
||||
assert urgent["type"] == "noul"
|
||||
assert set(urgent["probabilities"]) == {"yes", "no"}
|
||||
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
|
||||
pytest.approx(1.0, abs=1e-4)
|
||||
)
|
||||
assert urgent["noul"] == urgent["probabilities"]["yes"]
|
||||
assert urgent["confidence"] == pytest.approx(
|
||||
max(urgent["probabilities"].values())
|
||||
* body["diagnostics"]["urgent"]["label_mass"]
|
||||
)
|
||||
tone = body["answers"]["tone"]
|
||||
assert tone["type"] == "score"
|
||||
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
|
||||
assert set(tone["probabilities"]) == {"0", "1", "2"}
|
||||
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert tone["score"] == pytest.approx(
|
||||
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
|
||||
)
|
||||
assert 0 <= tone["score"] <= 2
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"questions,match",
|
||||
[
|
||||
(
|
||||
{
|
||||
"q": {
|
||||
"type": "choice",
|
||||
"criteria": {"a": None, "b": None},
|
||||
"depends_on": ["x"],
|
||||
}
|
||||
},
|
||||
"unknown field(s)",
|
||||
),
|
||||
(
|
||||
{
|
||||
f"q{i}": {"type": "choice", "criteria": {"a": None, "b": None}}
|
||||
for i in range(65)
|
||||
},
|
||||
"at most 64",
|
||||
),
|
||||
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
|
||||
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_validation_errors(server, questions, match):
|
||||
response = post(server, {"model": MODEL_NAME, "state": "x", "questions": questions})
|
||||
assert response.status_code == 400
|
||||
assert match in response.json()["error"]["message"]
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
@@ -410,198 +409,3 @@ def test_external_process_monitoring(api_server_args):
|
||||
mock_coordinator.shutdown()
|
||||
time.sleep(0.2)
|
||||
WORKER_RUNTIME_SECONDS = prev_worker_runtime
|
||||
|
||||
|
||||
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch):
|
||||
"""The Rust frontend command carries every non-default arg as JSON.
|
||||
Credentials must not reach the log line."""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
hf_token = "hf_TESTTOKEN123"
|
||||
api_key = "sk-TESTKEY456"
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--model", "org/model", "--hf-token", hf_token, "--api-key", api_key]
|
||||
)
|
||||
|
||||
class _FakeProc:
|
||||
pid = 4321
|
||||
returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
|
||||
with patch("vllm.v1.utils.logger.info") as log_info:
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
log_format, *log_args = log_info.call_args.args
|
||||
message = log_format % tuple(log_args)
|
||||
assert "Launching Rust frontend:" in message
|
||||
assert hf_token not in message
|
||||
assert api_key not in message
|
||||
assert '"hf_token": "***"' in message
|
||||
assert '"api_key": "***"' in message
|
||||
|
||||
|
||||
def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path):
|
||||
"""Config-only model selection must be forwarded to the Rust frontend."""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
config_path = tmp_path / "config.yml"
|
||||
config_path.write_text("model: org/model\n")
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--config", str(config_path)]
|
||||
)
|
||||
assert args.model == "org/model"
|
||||
assert args.model_tag is None
|
||||
|
||||
class _FakeProc:
|
||||
pid = 4321
|
||||
returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
|
||||
with caplog.at_level("INFO", logger="vllm.v1.utils"):
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
assert '"model_tag": "org/model"' in caplog.text
|
||||
|
||||
|
||||
def test_rust_frontend_inherits_grpc_listener(monkeypatch):
|
||||
"""The Python-bound gRPC socket is passed to Rust like the HTTP one."""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--model", "org/model", "--grpc-port", "50051"]
|
||||
)
|
||||
|
||||
class _FakeProc:
|
||||
pid = 4321
|
||||
returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
popen_calls = []
|
||||
|
||||
def fake_popen(cmd, **kwargs):
|
||||
popen_calls.append((cmd, kwargs))
|
||||
return _FakeProc()
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
input_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener("tcp://127.0.0.1:0", zmq.PULL)
|
||||
input_fd = input_listener.socket.fileno()
|
||||
output_fd = output_listener.socket.fileno()
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
grpc_sock=grpc_sock,
|
||||
)
|
||||
[(cmd, kwargs)] = popen_calls
|
||||
grpc_fd = grpc_sock.fileno()
|
||||
flag_index = cmd.index("--grpc-listen-fd")
|
||||
assert cmd[flag_index + 1] == str(grpc_fd)
|
||||
assert list(kwargs["pass_fds"]) == [
|
||||
sock.fileno(),
|
||||
input_fd,
|
||||
output_fd,
|
||||
grpc_fd,
|
||||
]
|
||||
finally:
|
||||
sock.close()
|
||||
grpc_sock.close()
|
||||
|
||||
|
||||
def test_rust_frontend_spawn_failure_cleans_up_listeners(tmp_path):
|
||||
"""No child owns the listeners when Popen fails, so the parent removes them."""
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--model", "org/model"]
|
||||
)
|
||||
input_listener = make_zmq_listener(f"ipc://{tmp_path}/input.sock", zmq.ROUTER)
|
||||
output_listener = make_zmq_listener(f"ipc://{tmp_path}/output.sock", zmq.PULL)
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
RustFrontendProcessManager(
|
||||
binary_path=str(tmp_path / "missing-vllm-rs"),
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_listener=input_listener,
|
||||
output_listener=output_listener,
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
for listener in (input_listener, output_listener):
|
||||
assert listener.socket.fileno() == -1
|
||||
assert not os.path.exists(listener.address.removeprefix("ipc://"))
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from starlette.datastructures import State
|
||||
|
||||
import vllm.entrypoints.launchers.render.app_state as app_state_mod
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class _CaptureKwargs:
|
||||
"""Stands in for OnlineRenderer/OnlineDerenderer; records init kwargs."""
|
||||
|
||||
captured: list[dict]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
type(self).captured.append(kwargs)
|
||||
|
||||
def warmup(self):
|
||||
pass
|
||||
|
||||
|
||||
def _render_cli_args(*argv: str) -> Namespace:
|
||||
"""Parse ``argv`` the way ``vllm launch render`` does, so new serve flags
|
||||
carry their defaults instead of needing a hand-maintained ``Namespace``."""
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(list(argv))
|
||||
if args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_app_state_uses_config_resolved_reasoning_parser(monkeypatch):
|
||||
"""gpt_oss defaults its reasoning parser through verify_and_update_config;
|
||||
the render server must pick up that resolved value like the main API
|
||||
server does, otherwise harmony markup leaks unparsed into derender output.
|
||||
"""
|
||||
model_config = ModelConfig("openai/gpt-oss-20b")
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
assert vllm_config.structured_outputs_config.reasoning_parser == "openai_gptoss"
|
||||
|
||||
class _Renderer(_CaptureKwargs):
|
||||
captured = []
|
||||
|
||||
class _Derenderer(_CaptureKwargs):
|
||||
captured = []
|
||||
|
||||
async def _noop_async(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_state_mod, "renderer_from_config", lambda cfg: object())
|
||||
monkeypatch.setattr(app_state_mod, "OnlineRenderer", _Renderer)
|
||||
monkeypatch.setattr(app_state_mod, "OnlineDerenderer", _Derenderer)
|
||||
monkeypatch.setattr(app_state_mod, "ServingTokenization", lambda *a, **kw: object())
|
||||
monkeypatch.setattr(app_state_mod, "init_render_state", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(app_state_mod, "init_endpoint_plugins_state", _noop_async)
|
||||
|
||||
args = _render_cli_args("openai/gpt-oss-20b")
|
||||
|
||||
await app_state_mod.init_render_app_state(vllm_config, State(), args)
|
||||
|
||||
assert _Renderer.captured[0]["reasoning_parser"] == "openai_gptoss"
|
||||
assert _Derenderer.captured[0]["reasoning_parser"] == "openai_gptoss"
|
||||
|
||||
|
||||
def test_explicit_reasoning_parser_flag_wins_over_model_default():
|
||||
"""An explicit --reasoning-parser must survive VllmConfig construction
|
||||
for models that define their own default (gpt_oss); the default only
|
||||
applies when the resolved value is empty."""
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
|
||||
engine_args = AsyncEngineArgs(
|
||||
model="openai/gpt-oss-20b", reasoning_parser="deepseek_r1"
|
||||
)
|
||||
model_config = engine_args.create_model_config()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
structured_outputs_config=engine_args.create_structured_outputs_config(),
|
||||
)
|
||||
|
||||
assert vllm_config.structured_outputs_config.reasoning_parser == "deepseek_r1"
|
||||
@@ -2,11 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.entrypoints.launchers.cli_args as cli_args_module
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
@@ -180,73 +178,6 @@ def test_multiple_valid_inputs(serve_parser):
|
||||
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
@pytest.mark.parametrize(
|
||||
"port, extra_args",
|
||||
[
|
||||
(50051, []),
|
||||
(0, ["--port", "0"]),
|
||||
(50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]),
|
||||
],
|
||||
)
|
||||
def test_rust_grpc_port_accepts_frontend_launch(
|
||||
vllm_parser, monkeypatch, port, extra_args
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
args = vllm_parser.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"--grpc-port",
|
||||
str(port),
|
||||
"--data-parallel-size",
|
||||
"8",
|
||||
"--data-parallel-size-local",
|
||||
"4",
|
||||
"--data-parallel-start-rank",
|
||||
"4",
|
||||
"--data-parallel-hybrid-lb",
|
||||
*extra_args,
|
||||
]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
assert args.grpc_port == port
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rust_enabled, extra_args, port",
|
||||
[
|
||||
(False, [], 50051),
|
||||
(False, ["--grpc"], 50051),
|
||||
(True, ["--grpc"], 50051),
|
||||
(True, ["--headless"], 50051),
|
||||
(True, ["--api-server-count", "0"], 50051),
|
||||
(True, ["--api-server-count", "-1"], 50051),
|
||||
(True, ["--data-parallel-multi-port-external-lb"], 50051),
|
||||
(True, ["--port", "50051"], 50051),
|
||||
(True, [], -1),
|
||||
(True, [], 65536),
|
||||
],
|
||||
)
|
||||
def test_rust_grpc_port_rejects_incompatible_launch(
|
||||
vllm_parser, monkeypatch, rust_enabled, extra_args, port
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0")
|
||||
args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args])
|
||||
error = (
|
||||
"--grpc and --grpc-port are mutually exclusive"
|
||||
if "--grpc" in extra_args
|
||||
else "--grpc-port"
|
||||
)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_rust_grpc_port_requires_serve_subcommand(monkeypatch):
|
||||
"""Direct Python render/API entrypoints must reject a Rust-only listener."""
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
parser = make_arg_parser(FlexibleArgumentParser())
|
||||
args = parser.parse_args(["--grpc-port", "50051"])
|
||||
with pytest.raises(ValueError, match="--grpc-port requires"):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_enable_auto_choice_fails_without_tool_call_parser(serve_parser):
|
||||
@@ -263,7 +194,7 @@ def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"mistral",
|
||||
"qwen3_coder",
|
||||
]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
@@ -280,7 +211,7 @@ def test_auto_tool_choice_with_reasoning_parser_passes(serve_parser):
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"mistral",
|
||||
"qwen3_coder",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
]
|
||||
@@ -333,48 +264,6 @@ def test_per_request_metrics_requires_log_stats(serve_parser):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def _build_launch_render_parser():
|
||||
"""Mirror `vllm launch render`.
|
||||
|
||||
`vllm/entrypoints/cli/main.py` parses subcommands with ``dest="subparser"``,
|
||||
and `LaunchSubcommandBase.add_cli_args` builds the component parser with
|
||||
`make_arg_parser`, so a launch component carries serve args under
|
||||
``args.subparser == "launch"``.
|
||||
"""
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
|
||||
launch_parser = subparsers.add_parser("launch")
|
||||
launch_subparsers = launch_parser.add_subparsers(
|
||||
required=True, dest="launch_component"
|
||||
)
|
||||
render_parser = launch_subparsers.add_parser("render")
|
||||
make_arg_parser(render_parser)
|
||||
return vllm_parser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def launch_render_parser():
|
||||
return _build_launch_render_parser()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, error",
|
||||
[
|
||||
(["--enable-auto-tool-choice"], TypeError),
|
||||
(["--grpc-port", "50051"], ValueError),
|
||||
],
|
||||
)
|
||||
def test_launch_render_validates_serve_args(
|
||||
launch_render_parser, monkeypatch, cli_args, error
|
||||
):
|
||||
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
args = launch_render_parser.parse_args(args=["launch", "render", *cli_args])
|
||||
assert args.subparser == "launch"
|
||||
with pytest.raises(error):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_subcommand_without_serve_args_skips_validation():
|
||||
"""A subcommand that does not use the serve parser is still skipped"""
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
@@ -456,26 +345,6 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
|
||||
assert args.default_chat_template_kwargs is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_kwargs,cohere_format,expected",
|
||||
[
|
||||
(None, "cmd3", {"cohere_format": "cmd3"}),
|
||||
(
|
||||
{"cohere_format": "cmd4", "custom": True},
|
||||
"cmd3",
|
||||
{"cohere_format": "cmd4", "custom": True},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
|
||||
args = SimpleNamespace(
|
||||
default_chat_template_kwargs=default_kwargs,
|
||||
cohere_format=cohere_format,
|
||||
)
|
||||
|
||||
assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||
"""Ensure invalid JSON raises an error."""
|
||||
with pytest.raises(SystemExit):
|
||||
|
||||
@@ -1,956 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.connections import HTTPConnection
|
||||
from vllm.entrypoints.launchers.run_batch import (
|
||||
BatchRequestOutput,
|
||||
download_bytes_from_url,
|
||||
upload_data,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
RERANKER_MODEL_NAME = "BAAI/bge-reranker-v2-m3"
|
||||
REASONING_MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
SPEECH_LARGE_MODEL_NAME = "openai/whisper-large-v3"
|
||||
SPEECH_SMALL_MODEL_NAME = "openai/whisper-small"
|
||||
|
||||
INPUT_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant.",
|
||||
},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are an unhelpful assistant.",
|
||||
},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-3",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": "NonExistModel",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are an unhelpful assistant.",
|
||||
},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-4",
|
||||
"method": "POST",
|
||||
"url": "/bad_url",
|
||||
"body": {
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are an unhelpful assistant.",
|
||||
},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-5",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"stream": "True",
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are an unhelpful assistant.",
|
||||
},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
INVALID_INPUT_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"invalid_field": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": CHAT_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are an unhelpful assistant."},
|
||||
{"role": "user", "content": "Hello world!"},
|
||||
],
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
INPUT_EMBEDDING_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/embeddings",
|
||||
"body": {
|
||||
"model": EMBEDDING_MODEL_NAME,
|
||||
"input": "You are a helpful assistant.",
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/embeddings",
|
||||
"body": {
|
||||
"model": EMBEDDING_MODEL_NAME,
|
||||
"input": "You are an unhelpful assistant.",
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-3",
|
||||
"method": "POST",
|
||||
"url": "/v1/embeddings",
|
||||
"body": {
|
||||
"model": EMBEDDING_MODEL_NAME,
|
||||
"input": "Hello world!",
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-4",
|
||||
"method": "POST",
|
||||
"url": "/v1/embeddings",
|
||||
"body": {
|
||||
"model": "NonExistModel",
|
||||
"input": "Hello world!",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
_SCORE_RERANK_DOCUMENTS = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
INPUT_SCORE_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/score",
|
||||
"body": {
|
||||
"model": RERANKER_MODEL_NAME,
|
||||
"queries": "What is the capital of France?",
|
||||
"documents": _SCORE_RERANK_DOCUMENTS,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/score",
|
||||
"body": {
|
||||
"model": RERANKER_MODEL_NAME,
|
||||
"queries": "What is the capital of France?",
|
||||
"documents": _SCORE_RERANK_DOCUMENTS,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
INPUT_RERANK_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/rerank",
|
||||
"body": {
|
||||
"model": RERANKER_MODEL_NAME,
|
||||
"query": "What is the capital of France?",
|
||||
"documents": _SCORE_RERANK_DOCUMENTS,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/rerank",
|
||||
"body": {
|
||||
"model": RERANKER_MODEL_NAME,
|
||||
"query": "What is the capital of France?",
|
||||
"documents": _SCORE_RERANK_DOCUMENTS,
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v2/rerank",
|
||||
"body": {
|
||||
"model": RERANKER_MODEL_NAME,
|
||||
"query": "What is the capital of France?",
|
||||
"documents": _SCORE_RERANK_DOCUMENTS,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
INPUT_REASONING_BATCH = "\n".join(
|
||||
json.dumps(req)
|
||||
for req in [
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": REASONING_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Solve this math problem: 2+2=?"},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"custom_id": "request-2",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": REASONING_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
MINIMAL_WAV_BASE64 = "UklGRigAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQQAAAAAAP9/"
|
||||
_EXACT_LIMIT_AUDIO = b"a" * MiB_bytes
|
||||
_OVERSIZED_BASE64_AUDIO = "A" * (4 * ((MiB_bytes + 2) // 3))
|
||||
_OVERSIZED_HTTP_AUDIO = b"b" * (MiB_bytes + 1)
|
||||
|
||||
|
||||
class _BatchHTTPServer(ThreadingHTTPServer):
|
||||
request_paths: list[str]
|
||||
|
||||
|
||||
class _BatchHTTPHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
server = self.server
|
||||
assert isinstance(server, _BatchHTTPServer)
|
||||
server.request_paths.append(self.path)
|
||||
|
||||
if self.path == "/exact-limit":
|
||||
self._send_body(_EXACT_LIMIT_AUDIO)
|
||||
return
|
||||
|
||||
if self.path == "/oversized-chunked":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self._write_body(_OVERSIZED_HTTP_AUDIO)
|
||||
return
|
||||
|
||||
self.send_error(404, "Unknown batch HTTP test path")
|
||||
|
||||
def _send_body(self, body: bytes) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self._write_body(body)
|
||||
|
||||
def _write_body(self, body: bytes) -> None:
|
||||
try:
|
||||
for offset in range(0, len(body), 64 * 1024):
|
||||
self.wfile.write(body[offset : offset + 64 * 1024])
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def batch_http_server() -> Generator[_BatchHTTPServer, None, None]:
|
||||
server = _BatchHTTPServer(("127.0.0.1", 0), _BatchHTTPHandler)
|
||||
server.request_paths = []
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _batch_http_url(server: _BatchHTTPServer, path: str) -> str:
|
||||
return f"http://localhost:{server.server_port}{path}"
|
||||
|
||||
|
||||
async def _close_async_connection(connection: HTTPConnection) -> None:
|
||||
if connection._async_client is not None:
|
||||
await connection._async_client.close()
|
||||
|
||||
|
||||
INPUT_TRANSCRIPTION_BATCH = (
|
||||
json.dumps(
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/audio/transcriptions",
|
||||
"body": {
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": f"data:audio/wav;base64,{MINIMAL_WAV_BASE64}",
|
||||
"response_format": "json",
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
INPUT_TRANSCRIPTION_HTTP_BATCH = (
|
||||
json.dumps(
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/audio/transcriptions",
|
||||
"body": {
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": AudioAsset("mary_had_lamb").url,
|
||||
"language": "en",
|
||||
"response_format": "json",
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
INPUT_TRANSLATION_BATCH = (
|
||||
json.dumps(
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/audio/translations",
|
||||
"body": {
|
||||
"model": SPEECH_SMALL_MODEL_NAME,
|
||||
"file_url": AudioAsset("mary_had_lamb").url,
|
||||
"response_format": "text",
|
||||
"language": "it",
|
||||
"to_language": "en",
|
||||
"temperature": 0.0,
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_TOOL_CALLING_BATCH = json.dumps(
|
||||
{
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": REASONING_MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is the weather in San Francisco?"},
|
||||
],
|
||||
"tools": [WEATHER_TOOL],
|
||||
"tool_choice": "required",
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_file():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write("")
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
EMBEDDING_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
assert contents.strip() == ""
|
||||
|
||||
|
||||
def test_completions():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
CHAT_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
# Ensure that the output format conforms to the openai api.
|
||||
# Validation should throw if the schema is wrong.
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
|
||||
def test_completions_invalid_input():
|
||||
"""Ensure that we fail when the input doesn't conform to the openai api."""
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INVALID_INPUT_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
CHAT_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode != 0, f"{proc=}"
|
||||
|
||||
|
||||
def test_embeddings():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_EMBEDDING_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
EMBEDDING_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
# Ensure that the output format conforms to the openai api.
|
||||
# Validation should throw if the schema is wrong.
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input_batch", [INPUT_SCORE_BATCH, INPUT_RERANK_BATCH])
|
||||
def test_score(input_batch):
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(input_batch)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
RERANKER_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
# Ensure that the output format conforms to the openai api.
|
||||
# Validation should throw if the schema is wrong.
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
# Ensure that there is no error in the response.
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
|
||||
def test_reasoning_parser():
|
||||
"""Test that reasoning_parser parameter works correctly in run_batch."""
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_REASONING_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
REASONING_MODEL_NAME,
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
# Ensure that the output format conforms to the openai api.
|
||||
# Validation should throw if the schema is wrong.
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
# Ensure that there is no error in the response.
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
# Check that reasoning is present and not empty
|
||||
reasoning = line_dict["response"]["body"]["choices"][0]["message"][
|
||||
"reasoning"
|
||||
]
|
||||
assert reasoning is not None
|
||||
assert len(reasoning) > 0
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_tool_calling():
|
||||
"""Test that tool calling works correctly in run_batch.
|
||||
Verifies that requests with tools return tool_calls in the response.
|
||||
"""
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_TOOL_CALLING_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
REASONING_MODEL_NAME,
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
if not line.strip(): # Skip empty lines
|
||||
continue
|
||||
# Ensure that the output format conforms to the openai api.
|
||||
# Validation should throw if the schema is wrong.
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
# Ensure that there is no error in the response.
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
# Check that tool_calls are present in the response
|
||||
# With tool_choice="required", the model must call a tool
|
||||
response_body = line_dict["response"]["body"]
|
||||
assert response_body is not None
|
||||
message = response_body["choices"][0]["message"]
|
||||
assert "tool_calls" in message
|
||||
tool_calls = message.get("tool_calls")
|
||||
# With tool_choice="required", tool_calls must be present and non-empty
|
||||
assert tool_calls is not None
|
||||
assert isinstance(tool_calls, list)
|
||||
assert len(tool_calls) > 0
|
||||
# Verify tool_calls have the expected structure
|
||||
for tool_call in tool_calls:
|
||||
assert "id" in tool_call
|
||||
assert "type" in tool_call
|
||||
assert tool_call["type"] == "function"
|
||||
assert "function" in tool_call
|
||||
assert "name" in tool_call["function"]
|
||||
assert "arguments" in tool_call["function"]
|
||||
# Verify the tool name matches our tool definition
|
||||
assert tool_call["function"]["name"] == "get_current_weather"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for download_bytes_from_url SSRF protection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_aiohttp_mocks(response_data: bytes = b"fake-data", status: int = 200):
|
||||
"""Create mock objects that simulate aiohttp.ClientSession context managers."""
|
||||
|
||||
async def iter_chunked(chunk_size: int):
|
||||
del chunk_size
|
||||
yield response_data
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = status
|
||||
mock_resp.content_length = len(response_data)
|
||||
mock_resp.content.iter_chunked = iter_chunked
|
||||
mock_resp.read = AsyncMock(return_value=response_data)
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=False)
|
||||
return mock_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_data_url_bypasses_domain_check():
|
||||
"""data: URLs must work regardless of the domain allowlist."""
|
||||
data_url = f"data:audio/wav;base64,{MINIMAL_WAV_BASE64}"
|
||||
result = await download_bytes_from_url(
|
||||
data_url, allowed_media_domains=["example.com"]
|
||||
)
|
||||
assert isinstance(result, bytes)
|
||||
assert len(result) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_rejects_disallowed_domain():
|
||||
"""HTTP URLs whose hostname is not in the allowlist must be rejected."""
|
||||
url = "https://evil.internal/secret"
|
||||
with pytest.raises(VLLMValidationError, match="allowed domains") as exc_info:
|
||||
await download_bytes_from_url(url, allowed_media_domains=["example.com"])
|
||||
# URL validation failures carry structured metadata for the frontend.
|
||||
assert exc_info.value.parameter == "url"
|
||||
assert exc_info.value.value == "evil.internal"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_rejects_unsupported_scheme():
|
||||
"""Unsupported URL schemes are rejected with structured metadata."""
|
||||
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme") as exc_info:
|
||||
await download_bytes_from_url("ftp://example.com/file")
|
||||
assert exc_info.value.parameter == "url"
|
||||
assert exc_info.value.value == "ftp"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_rejects_cloud_metadata_ip():
|
||||
"""Cloud metadata endpoints must be blocked when an allowlist is set."""
|
||||
url = "http://169.254.169.254/latest/meta-data/"
|
||||
with pytest.raises(VLLMValidationError, match="allowed domains"):
|
||||
await download_bytes_from_url(url, allowed_media_domains=["example.com"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_rejects_internal_ip():
|
||||
"""Private-range IPs must be blocked when an allowlist is set."""
|
||||
for internal_url in [
|
||||
"http://10.0.0.1/secret",
|
||||
"http://192.168.1.1/admin",
|
||||
"http://127.0.0.1:8080/internal",
|
||||
]:
|
||||
with pytest.raises(VLLMValidationError, match="allowed domains"):
|
||||
await download_bytes_from_url(
|
||||
internal_url, allowed_media_domains=["example.com"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_allows_permitted_domain():
|
||||
"""HTTP URLs whose hostname IS in the allowlist must be fetched."""
|
||||
url = "https://example.com/audio.wav"
|
||||
expected = b"audio-bytes"
|
||||
mock_session = _make_aiohttp_mocks(expected)
|
||||
|
||||
with patch(
|
||||
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
|
||||
return_value=mock_session,
|
||||
):
|
||||
result = await download_bytes_from_url(
|
||||
url, allowed_media_domains=["example.com"]
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_no_allowlist_permits_any_domain():
|
||||
"""Without an allowlist all HTTP URLs must be attempted (backward compat)."""
|
||||
url = "https://any-domain.example.org/file.wav"
|
||||
expected = b"some-data"
|
||||
mock_session = _make_aiohttp_mocks(expected)
|
||||
|
||||
with patch(
|
||||
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
|
||||
return_value=mock_session,
|
||||
):
|
||||
result = await download_bytes_from_url(url, allowed_media_domains=None)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_empty_allowlist_denies_all():
|
||||
"""An empty allowlist must deny all HTTP URLs (least privilege)."""
|
||||
url = "https://any-domain.example.org/file.wav"
|
||||
with pytest.raises(VLLMValidationError, match="allowed domains"):
|
||||
await download_bytes_from_url(url, allowed_media_domains=[])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_unsupported_scheme():
|
||||
"""Unsupported URL schemes must be rejected regardless of allowlist."""
|
||||
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme"):
|
||||
await download_bytes_from_url("ftp://example.com/file.wav")
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Unsupported URL scheme"):
|
||||
await download_bytes_from_url(
|
||||
"ftp://example.com/file.wav",
|
||||
allowed_media_domains=["example.com"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_backslash_bypass():
|
||||
"""Backslash-@ URL confusion must not bypass the allowlist.
|
||||
|
||||
urllib3.parse_url() and aiohttp/yarl disagree on backslash-before-@.
|
||||
The fix normalizes through urllib3 before handing to aiohttp.
|
||||
"""
|
||||
bypass_url = "http://allowed.example.com\\@evil.internal/secret"
|
||||
with pytest.raises(VLLMValidationError, match="allowed domains"):
|
||||
await download_bytes_from_url(
|
||||
bypass_url, allowed_media_domains=["evil.internal"]
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_bytes_allows_http_body_at_audio_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
batch_http_server: _BatchHTTPServer,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
connection = HTTPConnection()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.launchers.run_batch.global_http_connection",
|
||||
connection,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await download_bytes_from_url(
|
||||
_batch_http_url(batch_http_server, "/exact-limit")
|
||||
)
|
||||
assert result == _EXACT_LIMIT_AUDIO
|
||||
finally:
|
||||
await _close_async_connection(connection)
|
||||
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for upload_data retry behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_aiohttp_put_session(status: int = 200, body_text: str = ""):
|
||||
"""Mock an aiohttp.ClientSession whose PUT returns the given status."""
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = status
|
||||
mock_resp.text = AsyncMock(return_value=body_text)
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.put = MagicMock(return_value=mock_resp)
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=False)
|
||||
return mock_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [200, 201, 204])
|
||||
@pytest.mark.parametrize("from_file", [False, True])
|
||||
async def test_upload_data_uploads_once_on_success(status, from_file, tmp_path):
|
||||
"""Successful PUT responses must not retry either upload path."""
|
||||
session = _make_aiohttp_put_session(status=status)
|
||||
data_or_file = "payload"
|
||||
if from_file:
|
||||
path = tmp_path / "output.jsonl"
|
||||
path.write_text(data_or_file, encoding="utf-8")
|
||||
data_or_file = str(path)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
|
||||
return_value=session,
|
||||
),
|
||||
patch(
|
||||
"vllm.entrypoints.launchers.run_batch.asyncio.sleep",
|
||||
new_callable=AsyncMock,
|
||||
) as sleep,
|
||||
):
|
||||
await upload_data(
|
||||
"https://example.com/output.jsonl", data_or_file, from_file=from_file
|
||||
)
|
||||
assert session.put.call_count == 1
|
||||
sleep.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_data_error_includes_awaited_response_body():
|
||||
"""A failed upload must surface the awaited response body, not a coroutine."""
|
||||
session = _make_aiohttp_put_session(status=500, body_text="server-error-detail")
|
||||
with (
|
||||
patch(
|
||||
"vllm.entrypoints.launchers.run_batch.aiohttp.ClientSession",
|
||||
return_value=session,
|
||||
),
|
||||
patch(
|
||||
"vllm.entrypoints.launchers.run_batch.asyncio.sleep",
|
||||
AsyncMock(),
|
||||
),
|
||||
pytest.raises(Exception) as exc_info,
|
||||
):
|
||||
await upload_data(
|
||||
"https://example.com/output.jsonl", "payload", from_file=False
|
||||
)
|
||||
message = str(exc_info.value)
|
||||
assert "server-error-detail" in message
|
||||
assert "coroutine" not in message
|
||||
@@ -29,16 +29,6 @@ MODEL_CONFIGS = [
|
||||
"tensor_parallel_size": 1,
|
||||
"tokenizer": "Qwen/Qwen3-4B",
|
||||
},
|
||||
{
|
||||
"model": "mistralai/Mistral-7B-Instruct-v0.1",
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"max_model_len": 64,
|
||||
"max_num_batched_tokens": 64,
|
||||
"max_num_seqs": 64,
|
||||
"tensor_parallel_size": 1,
|
||||
"tokenizer_mode": "mistral",
|
||||
},
|
||||
# TODO: re-enable once these tests are run with V1
|
||||
# {
|
||||
# "model": "sentence-transformers/all-MiniLM-L12-v2",
|
||||
|
||||
@@ -16,7 +16,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
@@ -196,58 +195,6 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_render = ServingRender(models, online_renderer)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_render.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_render
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_render = _build_serving_render(mock_engine)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
result = await serving_render.render_chat_request(request)
|
||||
|
||||
assert result.token_ids == [1, 2, 3]
|
||||
assert (
|
||||
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream(include_usage: bool):
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_openai_decisions_text_questions(server):
|
||||
response = requests.post(
|
||||
server.url_for("v1/decisions"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "The package arrived with a broken screen.",
|
||||
"questions": [
|
||||
{"type": "predicate", "instructions": "Is the screen broken?"},
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "department",
|
||||
"instructions": "Which department should handle this?",
|
||||
"choices": [{"value": "returns"}, {"value": "billing"}],
|
||||
},
|
||||
{
|
||||
"type": "score",
|
||||
"name": "damage",
|
||||
"instructions": "Rate the damage.",
|
||||
"levels": [{"label": "undamaged"}, {"label": "broken"}],
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert set(body) == {"model", "answers", "usage"}
|
||||
predicate, choice_answer, score = body["answers"]
|
||||
assert predicate["type"] == "predicate"
|
||||
assert predicate["name"] is None
|
||||
assert 0 <= predicate["probability"] <= 1
|
||||
assert choice_answer["type"] == "choice"
|
||||
assert choice_answer["name"] == "department"
|
||||
assert [p["value"] for p in choice_answer["probabilities"]] == [
|
||||
"returns",
|
||||
"billing",
|
||||
]
|
||||
assert (
|
||||
choice_answer["choice"]
|
||||
== max(choice_answer["probabilities"], key=lambda p: p["probability"])["value"]
|
||||
)
|
||||
assert sum(
|
||||
p["probability"] for p in choice_answer["probabilities"]
|
||||
) == pytest.approx(1)
|
||||
assert score["type"] == "score"
|
||||
assert score["name"] == "damage"
|
||||
assert [p["label"] for p in score["probabilities"]] == ["undamaged", "broken"]
|
||||
assert score["score"] == pytest.approx(
|
||||
sum(p["value"] * p["probability"] for p in score["probabilities"])
|
||||
)
|
||||
usage = body["usage"]
|
||||
assert usage["output_tokens"] == 3
|
||||
assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"]
|
||||
|
||||
|
||||
def test_openai_decisions_rejects_images(server):
|
||||
response = requests.post(
|
||||
server.url_for("v1/decisions"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"questions": [{"type": "predicate", "instructions": "Is this red?"}],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "text input only" in response.json()["error"]["message"]
|
||||
@@ -2,9 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the token-offsets request/response protocol wiring:
|
||||
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
|
||||
serialization boundary. End-to-end behavior is covered by
|
||||
``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
|
||||
storage is not retested here.
|
||||
serialization boundary. Plain Pydantic field storage is not retested here.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -1,349 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared fixtures and helpers for the RL lifecycle test suite.
|
||||
|
||||
All test modules under this directory import from here to avoid duplication.
|
||||
|
||||
RFC: https://github.com/vllm-project/vllm/issues/45585
|
||||
PR: https://github.com/vllm-project/vllm/pull/45586
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model / server defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
MODEL_NAME = os.environ.get("VLLM_TEST_MODEL", "Qwen/Qwen3-0.6B")
|
||||
|
||||
|
||||
_BASE_ARGS = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--gpu-memory-utilization",
|
||||
"0.75",
|
||||
"--enable-sleep-mode",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
# Lightweight args for state-machine / protocol tests that don't need real
|
||||
# weights (avoids spending time downloading a 1B checkpoint in T0 tests).
|
||||
_DUMMY_ARGS = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"128",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--enable-sleep-mode",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server harness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _warm_up(url: str) -> None:
|
||||
"""Put one request through the engine before tests start timing things.
|
||||
|
||||
/health turns green before any request has travelled the request path, and
|
||||
that first pass costs seconds on a loaded machine.
|
||||
"""
|
||||
response = gen(url, max_tokens=4, timeout=120)
|
||||
assert ok(response), f"warm-up generation failed: {response}"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def server(
|
||||
extra_args=None,
|
||||
port: int = 8770,
|
||||
timeout: float = 180.0,
|
||||
dummy_weights: bool = False,
|
||||
):
|
||||
"""Launch a vLLM server with the dev router; yield its base URL.
|
||||
|
||||
Args:
|
||||
extra_args: Additional CLI flags appended after the base args.
|
||||
port: HTTP port to bind (caller is responsible for uniqueness).
|
||||
timeout: Seconds to wait for /health before giving up.
|
||||
dummy_weights: If True, use --load-format dummy (fast, no real weights).
|
||||
|
||||
"""
|
||||
env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"}
|
||||
base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.openai.api_server",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--port",
|
||||
str(port),
|
||||
"--served-model-name",
|
||||
"m",
|
||||
*(base + (extra_args or [])),
|
||||
]
|
||||
proc = subprocess.Popen(
|
||||
cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
||||
)
|
||||
url = f"http://localhost:{port}"
|
||||
try:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
err = (
|
||||
proc.stderr.read(4000).decode(errors="replace")
|
||||
if proc.stderr
|
||||
else ""
|
||||
)
|
||||
raise RuntimeError(f"vllm server exited during startup:\n{err}")
|
||||
with contextlib.suppress(Exception):
|
||||
if requests.get(f"{url}/health", timeout=3).status_code == 200:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
proc.terminate()
|
||||
raise RuntimeError("vllm server did not start in time")
|
||||
_warm_up(url)
|
||||
yield url
|
||||
finally:
|
||||
proc.terminate()
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
proc.wait(timeout=10)
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def gen(url, prompt="The capital of France is", max_tokens=8, timeout=30):
|
||||
"""Fire a /v1/completions request; return JSON or None on any error."""
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
return r.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def ok(resp) -> bool:
|
||||
"""True iff resp is a successful completion (has choices, no error key)."""
|
||||
return (
|
||||
resp is not None
|
||||
and "choices" in resp
|
||||
and bool(resp["choices"])
|
||||
and "error" not in resp
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — stream generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# First-token wait for a streaming request; loaded machines need the slack.
|
||||
STREAM_START_TIMEOUT = 20.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamResult:
|
||||
started: threading.Event = field(default_factory=threading.Event)
|
||||
done: threading.Event = field(default_factory=threading.Event)
|
||||
chunks: list[dict[str, Any]] = field(default_factory=list)
|
||||
finish_reason: str | None = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
def stream_completion(url: str, result: StreamResult, max_tokens: int) -> None:
|
||||
try:
|
||||
with requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": "Count upward slowly: one, two, three,",
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
"ignore_eos": True,
|
||||
"stream": True,
|
||||
},
|
||||
stream=True,
|
||||
timeout=(5, 60),
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
for line in response.iter_lines(decode_unicode=True):
|
||||
if not line or line == "data: [DONE]":
|
||||
continue
|
||||
assert line.startswith("data: ")
|
||||
chunk = json.loads(line.removeprefix("data: "))
|
||||
result.chunks.append(chunk)
|
||||
choice = chunk["choices"][0]
|
||||
if choice.get("text"):
|
||||
result.started.set()
|
||||
if choice.get("finish_reason") is not None:
|
||||
result.finish_reason = choice["finish_reason"]
|
||||
except Exception as error:
|
||||
result.error = error
|
||||
finally:
|
||||
result.done.set()
|
||||
|
||||
|
||||
def start_stream(url: str, max_tokens: int) -> tuple[StreamResult, threading.Thread]:
|
||||
result = StreamResult()
|
||||
thread = threading.Thread(
|
||||
target=stream_completion,
|
||||
args=(url, result, max_tokens),
|
||||
)
|
||||
thread.start()
|
||||
started = result.started.wait(timeout=STREAM_START_TIMEOUT)
|
||||
if not started or result.done.is_set():
|
||||
# Best-effort: on a stalled server these time out too, and would then
|
||||
# mask the assertions below.
|
||||
with contextlib.suppress(requests.RequestException):
|
||||
pause(url, mode="abort")
|
||||
resume(url)
|
||||
thread.join(timeout=10)
|
||||
assert started, (
|
||||
f"request did not start generating within {STREAM_START_TIMEOUT}s "
|
||||
f"(stream error: {result.error})"
|
||||
)
|
||||
assert not result.done.is_set(), "request completed before it could be paused"
|
||||
return result, thread
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — pause / resume
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pause(url, mode="abort", clear_cache=True):
|
||||
return requests.post(
|
||||
f"{url}/pause",
|
||||
params={"mode": mode, "clear_cache": clear_cache},
|
||||
timeout=15,
|
||||
).status_code
|
||||
|
||||
|
||||
def resume(url):
|
||||
return requests.post(f"{url}/resume", timeout=10).status_code
|
||||
|
||||
|
||||
def completion_with_cache_details(url: str, prompt: str) -> dict[str, Any]:
|
||||
response = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": "m",
|
||||
"prompt": prompt,
|
||||
"max_tokens": 8,
|
||||
"temperature": 0,
|
||||
"logprobs": 1,
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def golden_output(response: dict[str, Any]) -> dict[str, Any]:
|
||||
choice = response["choices"][0]
|
||||
usage = response["usage"]
|
||||
return {
|
||||
"text": choice["text"],
|
||||
"finish_reason": choice["finish_reason"],
|
||||
"tokens": choice["logprobs"]["tokens"],
|
||||
"prompt_tokens": usage["prompt_tokens"],
|
||||
"completion_tokens": usage["completion_tokens"],
|
||||
}
|
||||
|
||||
|
||||
def cached_tokens(response: dict[str, Any]) -> int:
|
||||
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — sleep / wake
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def sleep(url, level=1, mode="abort"):
|
||||
return requests.post(
|
||||
f"{url}/sleep", params={"level": level, "mode": mode}, timeout=15
|
||||
).status_code
|
||||
|
||||
|
||||
def wake(url, tags=None):
|
||||
params = {"tags": tags} if tags else {}
|
||||
return requests.post(f"{url}/wake_up", params=params, timeout=20).status_code
|
||||
|
||||
|
||||
def is_sleeping(url) -> bool:
|
||||
return requests.get(f"{url}/is_sleeping", timeout=5).json()["is_sleeping"]
|
||||
|
||||
|
||||
def is_paused(url) -> bool:
|
||||
return requests.get(f"{url}/is_paused", timeout=5).json()["is_paused"]
|
||||
|
||||
|
||||
def health(url) -> int:
|
||||
try:
|
||||
return requests.get(f"{url}/health", timeout=5).status_code
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP helpers — weight transfer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def start_weight_update(url, is_checkpoint_format=True):
|
||||
return requests.post(
|
||||
f"{url}/start_weight_update",
|
||||
json={"is_checkpoint_format": is_checkpoint_format},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
def finish_weight_update(url):
|
||||
return requests.post(f"{url}/finish_weight_update", timeout=10)
|
||||
|
||||
|
||||
def get_world_size(url, include_dp=True):
|
||||
return requests.get(
|
||||
f"{url}/get_world_size",
|
||||
params={"include_dp": include_dp},
|
||||
timeout=5,
|
||||
)
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
def decode_routed_experts(encoded: str | None) -> np.ndarray:
|
||||
assert encoded is not None
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
|
||||
assert routed_experts.ndim == 3
|
||||
num_tokens, num_layers, topk = routed_experts.shape
|
||||
assert num_tokens > 0
|
||||
assert num_layers == NUM_HIDDEN_LAYERS
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
return routed_experts
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enforce-eager",
|
||||
"--enable-return-routed-experts",
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routed_experts(server):
|
||||
"""Test that /v1/completions returns routed_experts when enabled."""
|
||||
async with server.get_async_client() as client:
|
||||
result = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Hello, world",
|
||||
max_tokens=10,
|
||||
temperature=0,
|
||||
extra_body={"return_token_ids": True},
|
||||
)
|
||||
|
||||
choice = result.model_dump()["choices"][0]
|
||||
|
||||
assert choice["token_ids"] is not None
|
||||
assert result.usage is not None
|
||||
routed_experts = decode_routed_experts(choice["routed_experts"])
|
||||
assert len(routed_experts) == (
|
||||
result.usage.prompt_tokens + len(choice["token_ids"]) - 1
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Malformed bodies on RL dev routes are 400s that never reach the engine."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.rl.online.api_router import router as rl_router
|
||||
from vllm.entrypoints.serve.dev.rpc.api_router import router as rpc_router
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self):
|
||||
self.calls: list[str] = []
|
||||
|
||||
def __getattr__(self, name):
|
||||
async def call(*args, **kwargs):
|
||||
self.calls.append(name)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = FastAPI()
|
||||
app.include_router(rl_router)
|
||||
app.include_router(rpc_router)
|
||||
init_exception_handler(app)
|
||||
app.state.engine_client = Engine()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def post(client, path, body):
|
||||
return client.post(
|
||||
path, content=json.dumps(body), headers={"content-type": "application/json"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/abort_requests",
|
||||
"/init_weight_transfer_engine",
|
||||
"/update_weights",
|
||||
"/collective_rpc",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("body", [[], 1, "x", None])
|
||||
def test_non_object_body_is_rejected(client, path, body):
|
||||
assert post(client, path, body).status_code == 400
|
||||
assert client.app.state.engine_client.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path,body",
|
||||
[
|
||||
("/init_weight_transfer_engine", {}),
|
||||
("/init_weight_transfer_engine", {"init_info": []}),
|
||||
("/init_weight_transfer_engine", {"init_info": 1}),
|
||||
("/update_weights", {}),
|
||||
("/update_weights", {"update_info": 1}),
|
||||
("/update_weights", {"update_info": [1, 2]}),
|
||||
("/update_weights", {"update_info": "x"}),
|
||||
],
|
||||
)
|
||||
def test_invalid_field_is_rejected_before_the_engine(client, path, body):
|
||||
# Reaching the engine would also abort an in-progress weight update.
|
||||
assert post(client, path, body).status_code == 400
|
||||
assert client.app.state.engine_client.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path,body,engine_call",
|
||||
[
|
||||
("/abort_requests", {"request_ids": ["a"]}, "abort"),
|
||||
(
|
||||
"/init_weight_transfer_engine",
|
||||
{"init_info": {}},
|
||||
"init_weight_transfer_engine",
|
||||
),
|
||||
("/update_weights", {"update_info": {"names": []}}, "update_weights"),
|
||||
("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
|
||||
("/collective_rpc", {"method": "m"}, "collective_rpc"),
|
||||
(
|
||||
"/shutdown_weight_transfer_engine",
|
||||
{},
|
||||
"shutdown_weight_transfer_engine",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_valid_body_reaches_the_engine(client, path, body, engine_call):
|
||||
assert post(client, path, body).status_code == 200
|
||||
assert client.app.state.engine_client.calls == [engine_call]
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# Its published sliding window is incompatible with AuxOutput retention.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enforce-eager",
|
||||
"--enable-return-routed-experts",
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1",
|
||||
},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_routed_experts(client):
|
||||
"""Test that /inference/v1/generate returns routed_experts when enabled."""
|
||||
prompt_token_ids = [1, 2, 3]
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": prompt_token_ids,
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
choice = data["choices"][0]
|
||||
|
||||
assert choice["routed_experts"] is not None
|
||||
assert choice["token_ids"] is not None
|
||||
|
||||
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
|
||||
assert routed_experts.ndim == 3
|
||||
num_tokens, num_layers, topk = routed_experts.shape
|
||||
assert num_tokens > 0
|
||||
assert num_layers == NUM_HIDDEN_LAYERS
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
assert num_tokens == len(prompt_token_ids) + len(choice["token_ids"]) - 1
|
||||
@@ -1,168 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the vLLM RL pause/resume lifecycle."""
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.entrypoints.rl.conftest import (
|
||||
cached_tokens,
|
||||
completion_with_cache_details,
|
||||
gen,
|
||||
golden_output,
|
||||
is_paused,
|
||||
ok,
|
||||
pause,
|
||||
resume,
|
||||
server,
|
||||
start_stream,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[False, True], ids=["MRV1", "MRV2"])
|
||||
def use_v2(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_url(use_v2):
|
||||
env_vars = {
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1" if use_v2 else "0",
|
||||
}
|
||||
|
||||
with (
|
||||
patch.dict(os.environ, env_vars),
|
||||
server(
|
||||
extra_args=[
|
||||
"--enable-prefix-caching",
|
||||
"--enable-prompt-tokens-details",
|
||||
]
|
||||
) as url,
|
||||
):
|
||||
yield url
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_unpaused_state(server_url):
|
||||
assert resume(server_url) == 200
|
||||
yield
|
||||
assert resume(server_url) == 200
|
||||
|
||||
|
||||
class TestPauseResume:
|
||||
def test_state_and_idempotency_across_cycles(self, server_url):
|
||||
assert not is_paused(server_url)
|
||||
|
||||
assert resume(server_url) == 200
|
||||
assert resume(server_url) == 200
|
||||
assert not is_paused(server_url)
|
||||
|
||||
for _ in range(2):
|
||||
for mode in ("abort", "wait", "keep"):
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert is_paused(server_url)
|
||||
|
||||
assert resume(server_url) == 200
|
||||
assert resume(server_url) == 200
|
||||
assert not is_paused(server_url)
|
||||
|
||||
def test_invalid_mode_preserves_state(self, server_url):
|
||||
for paused in (False, True):
|
||||
if paused:
|
||||
assert pause(server_url) == 200
|
||||
assert is_paused(server_url) is paused
|
||||
|
||||
response = requests.post(
|
||||
f"{server_url}/pause",
|
||||
params={"mode": "invalid"},
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["param"] == "query.mode"
|
||||
assert is_paused(server_url) is paused
|
||||
|
||||
assert resume(server_url) == 200
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "max_tokens", "inflight_finish_reason"),
|
||||
[
|
||||
pytest.param("abort", 256, "abort", id="abort"),
|
||||
pytest.param("wait", 256, "length", id="wait"),
|
||||
pytest.param("keep", 256, "length", id="keep"),
|
||||
],
|
||||
)
|
||||
def test_mode_request_lifecycle(
|
||||
self,
|
||||
server_url,
|
||||
mode,
|
||||
max_tokens,
|
||||
inflight_finish_reason,
|
||||
):
|
||||
inflight, inflight_thread = start_stream(server_url, max_tokens)
|
||||
new_result: dict[str, Any] = {}
|
||||
new_done = threading.Event()
|
||||
|
||||
def _new_request():
|
||||
new_result["response"] = gen(server_url, max_tokens=4, timeout=60)
|
||||
new_done.set()
|
||||
|
||||
new_thread = threading.Thread(target=_new_request)
|
||||
try:
|
||||
assert pause(server_url, mode=mode) == 200
|
||||
assert is_paused(server_url)
|
||||
|
||||
if mode in ("abort", "wait"):
|
||||
assert inflight.done.is_set()
|
||||
else:
|
||||
chunks_after_pause = len(inflight.chunks)
|
||||
assert not inflight.done.wait(timeout=5)
|
||||
assert len(inflight.chunks) == chunks_after_pause, (
|
||||
"in-flight request continued generating in keep mode"
|
||||
)
|
||||
|
||||
new_thread.start()
|
||||
assert not new_done.wait(timeout=0.3), (
|
||||
"new request completed while generation was paused"
|
||||
)
|
||||
finally:
|
||||
assert resume(server_url) == 200
|
||||
inflight_thread.join(timeout=30)
|
||||
if new_thread.ident is not None:
|
||||
new_thread.join(timeout=30)
|
||||
|
||||
assert not inflight_thread.is_alive()
|
||||
assert inflight.error is None
|
||||
assert inflight.finish_reason == inflight_finish_reason
|
||||
assert not new_thread.is_alive()
|
||||
assert ok(new_result.get("response"))
|
||||
|
||||
def test_clear_cache_preserves_output_and_controls_prefix_cache(self, server_url):
|
||||
prompt = (
|
||||
"Paris is the capital of France. Berlin is the capital of Germany. "
|
||||
) * 20
|
||||
|
||||
assert pause(server_url, clear_cache=True) == 200
|
||||
assert resume(server_url) == 200
|
||||
baseline = completion_with_cache_details(server_url, prompt)
|
||||
warmed = completion_with_cache_details(server_url, prompt)
|
||||
|
||||
assert cached_tokens(baseline) == 0
|
||||
assert cached_tokens(warmed) > 0
|
||||
|
||||
assert pause(server_url, clear_cache=False) == 200
|
||||
assert resume(server_url) == 200
|
||||
preserved = completion_with_cache_details(server_url, prompt)
|
||||
assert golden_output(preserved) == golden_output(baseline)
|
||||
assert cached_tokens(preserved) > 0
|
||||
|
||||
assert pause(server_url, clear_cache=True) == 200
|
||||
assert resume(server_url) == 200
|
||||
cleared = completion_with_cache_details(server_url, prompt)
|
||||
assert golden_output(cleared) == golden_output(baseline)
|
||||
assert cached_tokens(cleared) == 0
|
||||
@@ -1,306 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.instrumentator.metrics import (
|
||||
attach_router as attach_metrics_router,
|
||||
)
|
||||
from vllm.v1.metrics import prometheus as prometheus_metrics
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sleep_route_app(monkeypatch):
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
|
||||
lambda: metrics,
|
||||
)
|
||||
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
return app, metrics
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("level", [0, 1, 2])
|
||||
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
|
||||
app, _ = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/sleep", params={"level": level})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "sleeping", "level": level}
|
||||
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_param"),
|
||||
[
|
||||
("level=invalid", "query.level"),
|
||||
("level=-1", "query.level"),
|
||||
("level=3", "query.level"),
|
||||
("mode=invalid", "query.mode"),
|
||||
],
|
||||
)
|
||||
def test_sleep_route_rejects_invalid_query_before_dispatch(
|
||||
sleep_route_app, query, expected_param
|
||||
):
|
||||
app, metrics = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/sleep?{query}")
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["param"] == expected_param
|
||||
app.state.engine_client.sleep.assert_not_awaited()
|
||||
assert list(metrics.duration.collect()[0].samples) == []
|
||||
assert list(metrics.in_flight.collect()[0].samples) == []
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("fully_awake", "tags", "expected"),
|
||||
[
|
||||
(True, "", {"status": "awake", "tags": None}),
|
||||
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
|
||||
],
|
||||
)
|
||||
def test_wake_route_maps_engine_result_without_state_query(
|
||||
sleep_route_app, fully_awake, tags, expected
|
||||
):
|
||||
app, _ = sleep_route_app
|
||||
app.state.engine_client.wake_up.return_value = fully_awake
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/wake_up{tags}")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == expected
|
||||
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
|
||||
app.state.engine_client.is_sleeping.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(sleep_route_app, fails):
|
||||
app, metrics = sleep_route_app
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
if not fails:
|
||||
assert response.json() == {"status": "kv_cache_released"}
|
||||
release.assert_awaited_once_with()
|
||||
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
|
||||
assert _duration_count(metrics, "release_kv_cache_memory") == 1
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
|
||||
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
|
||||
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
with metrics.record(operation):
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 1
|
||||
with pytest.raises(exception), metrics.record(operation):
|
||||
raise exception()
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 0
|
||||
assert metrics.duration.labels(operation)._sum.get() >= 0
|
||||
assert _duration_count(metrics, operation) == 2
|
||||
|
||||
|
||||
def _duration_count(metrics, operation):
|
||||
return next(
|
||||
sample.value
|
||||
for sample in metrics.duration.collect()[0].samples
|
||||
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("path", "method", "operation", "status", "fails"),
|
||||
[
|
||||
("/sleep", "POST", "sleep", 200, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
|
||||
("/wake_up", "POST", "wake", 200, False),
|
||||
("/is_sleeping", "GET", None, 200, False),
|
||||
("/sleep?level=3", "POST", None, 400, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
|
||||
],
|
||||
)
|
||||
def test_sleep_routes_visible_on_production_metrics_endpoint(
|
||||
monkeypatch, path, method, operation, status, fails
|
||||
):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
|
||||
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
|
||||
monkeypatch.setattr(sleep_metrics, "_metrics", None)
|
||||
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
app.state.engine_client.wake_up.return_value = True
|
||||
app.state.engine_client.is_sleeping.return_value = False
|
||||
if fails:
|
||||
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
|
||||
"engine failed"
|
||||
)
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
attach_metrics_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
assert client.request(method, path).status_code == status
|
||||
response = client.get("/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
samples = [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
]
|
||||
assert any(
|
||||
sample.name == "http_requests_total"
|
||||
and sample.labels
|
||||
== {
|
||||
"handler": path.split("?")[0],
|
||||
"method": method,
|
||||
"status": f"{status // 100}xx",
|
||||
}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
)
|
||||
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operations_in_flight"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 0
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enable-sleep-mode",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_SERVER_DEV_MODE": "1", "CUDA_VISIBLE_DEVICES": "0"},
|
||||
) as remote_server:
|
||||
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
|
||||
assert response.status_code == 200
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 0
|
||||
assert weights_offloaded == 1
|
||||
assert discard_all == 0
|
||||
|
||||
response = requests.post(remote_server.url_for("wake_up"))
|
||||
assert response.status_code == 200
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is False
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 1
|
||||
assert weights_offloaded == 0
|
||||
assert discard_all == 0
|
||||
|
||||
# test wake up with tags
|
||||
response = requests.post(remote_server.url_for("sleep"), params={"level": "1"})
|
||||
assert response.status_code == 200
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("wake_up"), params={"tags": ["weights"]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Partial wake keeps the engine sleeping.
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is True
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("wake_up"), params={"tags": ["kv_cache"]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
response = requests.get(remote_server.url_for("is_sleeping"))
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("is_sleeping") is False
|
||||
|
||||
# check sleep metrics
|
||||
response = requests.get(remote_server.url_for("metrics"))
|
||||
assert response.status_code == 200
|
||||
awake, weights_offloaded, discard_all = _get_sleep_metrics_from_api(response)
|
||||
assert awake == 1
|
||||
assert weights_offloaded == 0
|
||||
assert discard_all == 0
|
||||
|
||||
|
||||
def _get_sleep_metrics_from_api(response: requests.Response):
|
||||
"""Return (awake, weights_offloaded, discard_all)."""
|
||||
awake, weights_offloaded, discard_all = None, None, None
|
||||
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
if family.name == "vllm:engine_sleep_state":
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:engine_sleep_state":
|
||||
for label_name, label_value in sample.labels.items():
|
||||
if label_value == "awake":
|
||||
awake = sample.value
|
||||
elif label_value == "weights_offloaded":
|
||||
weights_offloaded = sample.value
|
||||
elif label_value == "discard_all":
|
||||
discard_all = sample.value
|
||||
|
||||
assert awake is not None
|
||||
assert weights_offloaded is not None
|
||||
assert discard_all is not None
|
||||
|
||||
return awake, weights_offloaded, discard_all
|
||||
@@ -1,3 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for weight transfer APIs via LLM class.
|
||||
|
||||
These tests use a mock weight transfer engine to verify that the API
|
||||
calls the correct methods with the right arguments, without requiring
|
||||
actual NCCL communication.
|
||||
"""
|
||||
|
||||
import os
|
||||
import weakref
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm.config import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateInfo,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
|
||||
# Use a tiny model for fast testing
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
# --- Mock Weight Transfer Engine ---
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockInitInfo(WeightTransferInitInfo):
|
||||
"""Mock initialization info."""
|
||||
|
||||
test_param: str = "test"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Mock update info."""
|
||||
|
||||
names: list[str] | None = None
|
||||
dtype_names: list[str] | None = None
|
||||
shapes: list[list[int]] | None = None
|
||||
|
||||
|
||||
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
|
||||
"""Mock weight transfer engine that tracks method calls."""
|
||||
|
||||
init_info_cls = MockInitInfo
|
||||
update_info_cls = MockUpdateInfo
|
||||
|
||||
# Class-level tracking for verification across processes
|
||||
init_transfer_engine_called: bool = False
|
||||
start_called: bool = False
|
||||
receive_weights_called: bool = False
|
||||
finish_called: bool = False
|
||||
shutdown_called: bool = False
|
||||
last_init_info: MockInitInfo | None = None
|
||||
last_update_info: MockUpdateInfo | None = None
|
||||
|
||||
def __init__(self, config, vllm_config, device, model):
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
# Reset tracking on init
|
||||
MockWeightTransferEngine.init_transfer_engine_called = False
|
||||
MockWeightTransferEngine.start_called = False
|
||||
MockWeightTransferEngine.receive_weights_called = False
|
||||
MockWeightTransferEngine.finish_called = False
|
||||
MockWeightTransferEngine.shutdown_called = False
|
||||
MockWeightTransferEngine.last_init_info = None
|
||||
MockWeightTransferEngine.last_update_info = None
|
||||
|
||||
def init_transfer_engine(self, init_info: MockInitInfo) -> None:
|
||||
MockWeightTransferEngine.init_transfer_engine_called = True
|
||||
MockWeightTransferEngine.last_init_info = init_info
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
MockWeightTransferEngine.start_called = True
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
MockWeightTransferEngine.finish_called = True
|
||||
|
||||
def receive_weights(self, update_info: MockUpdateInfo) -> None:
|
||||
MockWeightTransferEngine.receive_weights_called = True
|
||||
MockWeightTransferEngine.last_update_info = update_info
|
||||
|
||||
def shutdown(self) -> None:
|
||||
MockWeightTransferEngine.shutdown_called = True
|
||||
|
||||
|
||||
def mock_create_engine(config, vllm_config, device, model):
|
||||
"""Mock factory function that returns our mock engine."""
|
||||
return MockWeightTransferEngine(config, vllm_config, device, model)
|
||||
|
||||
|
||||
# --- Tests ---
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_get_world_size_tp1(vllm_runner):
|
||||
"""Test world_size is correctly configured for TP=1."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
) as runner:
|
||||
world_size = runner.llm.llm_engine.vllm_config.parallel_config.world_size
|
||||
assert world_size == 1
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
|
||||
"""Test that init_weight_transfer_engine calls the engine's
|
||||
init_transfer_engine method."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
# Run in-process so mock.patch works (spawn won't inherit the mock)
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||
mock_create_engine,
|
||||
),
|
||||
vllm_runner(
|
||||
MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
) as runner,
|
||||
):
|
||||
llm = weakref.proxy(runner.llm)
|
||||
|
||||
# Verify engine was created
|
||||
def check_engine_exists(self):
|
||||
return self.weight_transfer_engine is not None
|
||||
|
||||
results = llm.collective_rpc(check_engine_exists)
|
||||
assert all(results), "Weight transfer engine should be initialized"
|
||||
|
||||
# Call init_weight_transfer_engine
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "hello"})
|
||||
)
|
||||
|
||||
# Verify init_transfer_engine was called on the engine
|
||||
def check_init_called(self):
|
||||
engine = self.weight_transfer_engine
|
||||
return (
|
||||
engine.init_transfer_engine_called,
|
||||
engine.last_init_info.test_param if engine.last_init_info else None,
|
||||
)
|
||||
|
||||
results = llm.collective_rpc(check_init_called)
|
||||
for called, param in results:
|
||||
assert called, "init_transfer_engine should have been called"
|
||||
assert param == "hello", f"Expected 'hello', got {param}"
|
||||
|
||||
llm.shutdown_weight_transfer_engine()
|
||||
assert all(
|
||||
llm.collective_rpc(
|
||||
lambda worker: worker.weight_transfer_engine.shutdown_called
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_update_weights_calls_engine(vllm_runner):
|
||||
"""Test that update_weights calls the engine's receive_weights method."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
# Run in-process so mock.patch works (spawn won't inherit the mock)
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||
mock_create_engine,
|
||||
),
|
||||
vllm_runner(
|
||||
MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
) as runner,
|
||||
):
|
||||
llm = weakref.proxy(runner.llm)
|
||||
|
||||
# First init the weight transfer
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "init"})
|
||||
)
|
||||
llm.start_weight_update()
|
||||
|
||||
# Call update_weights
|
||||
test_names = ["layer.weight", "layer.bias"]
|
||||
test_dtypes = ["float32", "float32"]
|
||||
test_shapes = [[10, 10], [10]]
|
||||
|
||||
llm.update_weights(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info={
|
||||
"names": test_names,
|
||||
"dtype_names": test_dtypes,
|
||||
"shapes": test_shapes,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Verify receive_weights was called with correct info
|
||||
def check_update_called(self):
|
||||
engine = self.weight_transfer_engine
|
||||
if not engine.receive_weights_called:
|
||||
return False, None, None, None
|
||||
info = engine.last_update_info
|
||||
return (True, info.names, info.dtype_names, info.shapes)
|
||||
|
||||
results = llm.collective_rpc(check_update_called)
|
||||
for called, names, dtypes, shapes in results:
|
||||
assert called, "receive_weights should have been called"
|
||||
assert names == test_names
|
||||
assert dtypes == test_dtypes
|
||||
assert shapes == test_shapes
|
||||
|
||||
llm.finish_weight_update()
|
||||
assert llm.get_weight_version() == "default"
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_full_weight_transfer_flow(vllm_runner):
|
||||
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
# Run in-process so mock.patch works (spawn won't inherit the mock)
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||
mock_create_engine,
|
||||
),
|
||||
vllm_runner(
|
||||
MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
) as runner,
|
||||
):
|
||||
llm = weakref.proxy(runner.llm)
|
||||
|
||||
assert llm.get_weight_version() == "default"
|
||||
|
||||
# Step 1: Initialize weight transfer engine
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "flow_test"})
|
||||
)
|
||||
|
||||
# Step 2: Start weight update
|
||||
llm.start_weight_update()
|
||||
|
||||
# Step 3: Update weights
|
||||
llm.update_weights(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info={
|
||||
"names": ["test.weight"],
|
||||
"dtype_names": ["bfloat16"],
|
||||
"shapes": [[100, 100]],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert llm.get_weight_version() == "default"
|
||||
|
||||
# Step 4: Finish weight update
|
||||
llm.finish_weight_update("step-42")
|
||||
|
||||
assert llm.get_weight_version() == "step-42"
|
||||
|
||||
llm.update_weight_version("manual-version")
|
||||
assert llm.get_weight_version() == "manual-version"
|
||||
|
||||
# Verify the full flow completed
|
||||
def check_flow(self):
|
||||
engine = self.weight_transfer_engine
|
||||
return {
|
||||
"init_called": engine.init_transfer_engine_called,
|
||||
"update_called": engine.receive_weights_called,
|
||||
"init_param": (
|
||||
engine.last_init_info.test_param if engine.last_init_info else None
|
||||
),
|
||||
"update_names": (
|
||||
engine.last_update_info.names if engine.last_update_info else None
|
||||
),
|
||||
}
|
||||
|
||||
results = llm.collective_rpc(check_flow)
|
||||
for result in results:
|
||||
assert result["init_called"], "init_transfer_engine should be called"
|
||||
assert result["update_called"], "receive_weights should be called"
|
||||
assert result["init_param"] == "flow_test"
|
||||
assert result["update_names"] == ["test.weight"]
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_weight_transfer_config_backend(vllm_runner):
|
||||
"""Test that WeightTransferConfig backend is properly configured."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
# Test with nccl backend
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
) as runner:
|
||||
config = runner.llm.llm_engine.vllm_config.weight_transfer_config
|
||||
assert config is not None
|
||||
assert config.backend == "nccl"
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
|
||||
from vllm.entrypoints.rl.online import api_router
|
||||
from vllm.entrypoints.rl.online import metrics as rl_metrics
|
||||
from vllm.entrypoints.rl.online.metrics import WeightOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.v1.metrics.prometheus import get_prometheus_registry
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds_count"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
|
||||
|
||||
def sample(registry, name, operation):
|
||||
return registry.get_sample_value(name, {"operation": operation})
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, engine):
|
||||
self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine))
|
||||
|
||||
|
||||
class Engine:
|
||||
"""Holds every engine call open until the test releases it."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
|
||||
def __getattr__(self, name):
|
||||
async def call(*args, **kwargs):
|
||||
self.started.set()
|
||||
await self.release.wait()
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.setattr(rl_metrics, "_metrics", WeightOperationMetrics(registry))
|
||||
return registry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
def test_record_releases_in_flight_and_observes_duration(registry, fail):
|
||||
metrics = rl_metrics.weight_operation_metrics()
|
||||
with (
|
||||
pytest.raises(RuntimeError) if fail else nullcontext(),
|
||||
metrics.record("update"),
|
||||
):
|
||||
assert sample(registry, IN_FLIGHT, "update") == 1
|
||||
if fail:
|
||||
raise RuntimeError
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"operation,route,body",
|
||||
[
|
||||
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
|
||||
("shutdown", api_router.shutdown_weight_transfer_engine, None),
|
||||
("start", api_router.start_weight_update, None),
|
||||
("start_draft", api_router.start_draft_weight_update, None),
|
||||
("update", api_router.update_weights, {"update_info": {}}),
|
||||
("finish", api_router.finish_weight_update, None),
|
||||
],
|
||||
)
|
||||
async def test_route_records_its_operation(registry, operation, route, body):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(route(Request(engine), **(body or {})))
|
||||
await engine.started.wait()
|
||||
assert sample(registry, IN_FLIGHT, operation) == 1
|
||||
engine.release.set()
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, operation) == 0
|
||||
assert sample(registry, DURATION, operation) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_route_releases_in_flight(registry):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(
|
||||
api_router.update_weights(Request(engine), update_info={})
|
||||
)
|
||||
await engine.started.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
HOLD_OPERATION = """
|
||||
import sys
|
||||
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
|
||||
with weight_operation_metrics().record("update"):
|
||||
print("entered", flush=True)
|
||||
sys.stdin.readline()
|
||||
"""
|
||||
|
||||
|
||||
def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch):
|
||||
"""Two API-server processes each hold an operation; /metrics sums them."""
|
||||
monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path))
|
||||
servers = [
|
||||
subprocess.Popen(
|
||||
[sys.executable, "-c", HOLD_OPERATION],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
env=os.environ.copy(),
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
try:
|
||||
for server in servers:
|
||||
assert server.stdout is not None
|
||||
assert server.stdout.readline().strip() == "entered"
|
||||
assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2
|
||||
finally:
|
||||
for server in servers:
|
||||
server.communicate("\n", timeout=60)
|
||||
|
||||
scraped = get_prometheus_registry()
|
||||
assert sample(scraped, IN_FLIGHT, "update") == 0
|
||||
assert sample(scraped, DURATION, "update") == 2
|
||||
|
||||
|
||||
def test_rejected_request_is_not_recorded(registry):
|
||||
app = FastAPI()
|
||||
app.include_router(api_router.router)
|
||||
init_exception_handler(app)
|
||||
app.state.engine_client = Engine()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
response = TestClient(app).post("/update_weights", json={})
|
||||
assert response.status_code == 400
|
||||
assert sample(registry, DURATION, "update") is None
|
||||
|
||||
|
||||
def test_default_recorder_is_created_lazily_on_the_default_registry():
|
||||
"""The default recorder registers on the process registry only when first used.
|
||||
|
||||
Runs in a fresh interpreter: the registry is process-global, so a route test
|
||||
elsewhere in the session may already have registered these collectors, and
|
||||
"not registered yet" would then be unobservable.
|
||||
"""
|
||||
probe = """
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from vllm.entrypoints.rl.online import metrics as rl_metrics
|
||||
|
||||
NAMES = [
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
]
|
||||
assert rl_metrics._metrics is None
|
||||
assert not set(NAMES) & set(REGISTRY._names_to_collectors)
|
||||
|
||||
metrics = rl_metrics.weight_operation_metrics()
|
||||
assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration
|
||||
assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight
|
||||
assert rl_metrics.weight_operation_metrics() is metrics
|
||||
print("lazy registration ok")
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", probe],
|
||||
cwd=Path(__file__).parents[3],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "lazy registration ok" in result.stdout
|
||||
@@ -1,880 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Round trip parity CI: render -> generate -> derender -> render == render.
|
||||
|
||||
Pins the coupled parsing path (`/v1/chat/completions` on a normal GPU
|
||||
server which parses generated tokens incrementally as they stream out)
|
||||
against the disaggregated path (`OnlineDerenderer.derender_chat` via
|
||||
`/v1/chat/completions/derender` which parses the same tokens all at once,
|
||||
out of process). A standard `vllm serve` GPU server mounts both, so one
|
||||
real generation lets both parsers run on identical input.
|
||||
|
||||
Both paths consume the same generated token IDs (extracted from the coupled
|
||||
response's `token_ids` via `return_token_ids=True`), so generation
|
||||
nondeterminism is irrelevant. The only variable under test is whether the
|
||||
two parsing code paths agree. Parity is asserted unconditionally as only the
|
||||
stronger per case assertions (e.g. "a tool call was produced") are gated
|
||||
behind the marker actually having been emitted since a 1.5B model is not
|
||||
guaranteed to emit `<think>` / `<tool_call>`.
|
||||
|
||||
The streaming cases do the same for `stream=true`. Each coupled chunk's
|
||||
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
|
||||
through the chunked derender path with `stream_state` threaded across calls
|
||||
the way a client would, so both parsers see the same chunk boundaries.
|
||||
|
||||
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
|
||||
against `/v1/completions/derender` on the same token IDs, batch and streaming,
|
||||
plus logprob resolution and stop string handling.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.scale_out.derender.utils import (
|
||||
assemble_stream,
|
||||
stream_chat_derender,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
ARGS = [
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _coupled(client: httpx.AsyncClient, messages: list[dict], **extra) -> dict:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"messages": messages,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"return_token_ids": True,
|
||||
**extra,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def _disagg(
|
||||
client: httpx.AsyncClient,
|
||||
output_ids: list[int],
|
||||
prompt_tokens: int,
|
||||
finish_reason: str,
|
||||
chat_request: dict,
|
||||
logprobs: dict | None = None,
|
||||
) -> dict:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"generate_response": {
|
||||
"request_id": "parity",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": output_ids,
|
||||
"finish_reason": finish_reason,
|
||||
"logprobs": logprobs,
|
||||
}
|
||||
],
|
||||
},
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"chat_request": {"return_token_ids": True, **chat_request},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
|
||||
"""[(name, json normalized args)] so key ordering / whitespace don't
|
||||
cause false negatives."""
|
||||
return [
|
||||
(tc["function"]["name"], json.loads(tc["function"]["arguments"]))
|
||||
for tc in (response_choice["message"].get("tool_calls") or [])
|
||||
]
|
||||
|
||||
|
||||
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
|
||||
return [
|
||||
_tool_call_id_shape(tc["id"])
|
||||
for tc in (response_choice["message"].get("tool_calls") or [])
|
||||
]
|
||||
|
||||
|
||||
def _engine_finish_reason(coupled_finish_reason: str) -> str:
|
||||
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
|
||||
gets what the engine reported and has to reapply the rewrite itself."""
|
||||
if coupled_finish_reason == "tool_calls":
|
||||
return "stop"
|
||||
return coupled_finish_reason
|
||||
|
||||
|
||||
def _assert_parity(coupled: dict, disagg: dict) -> None:
|
||||
"""Both paths saw the same tokens, so they must agree unconditionally."""
|
||||
c, d = coupled["choices"][0], disagg["choices"][0]
|
||||
assert d["message"]["content"] == c["message"]["content"]
|
||||
assert d["message"].get("reasoning") == c["message"].get("reasoning")
|
||||
assert _tool_sig(d) == _tool_sig(c)
|
||||
assert _tool_id_shapes(d) == _tool_id_shapes(c)
|
||||
assert d["finish_reason"] == c["finish_reason"]
|
||||
assert d["token_ids"] == c["token_ids"]
|
||||
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
|
||||
assert disagg["usage"]["completion_tokens"] == len(c["token_ids"])
|
||||
|
||||
|
||||
async def _run_parity_case(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[dict, dict]:
|
||||
"""Run the coupled request then feed its generated tokens into the
|
||||
disaggregated derender endpoint. Returns (coupled, disagg)."""
|
||||
coupled = await _coupled(client, messages, **extra)
|
||||
ch = coupled["choices"][0]
|
||||
chat_request = {"model": MODEL, "messages": messages, **extra}
|
||||
disagg = await _disagg(
|
||||
client,
|
||||
ch["token_ids"],
|
||||
coupled["usage"]["prompt_tokens"],
|
||||
_engine_finish_reason(ch["finish_reason"]),
|
||||
chat_request,
|
||||
)
|
||||
return coupled, disagg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parity cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_plain(client):
|
||||
"""Plain detokenization parity. No reasoning/tool markers involved."""
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
coupled, disagg = await _run_parity_case(client, messages)
|
||||
_assert_parity(coupled, disagg)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_reasoning(client):
|
||||
"""Reasoning/content split parity for <think>...</think> outputs."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, disagg = await _run_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
_assert_parity(coupled, disagg)
|
||||
|
||||
if not coupled["choices"][0]["message"].get("reasoning"):
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
assert disagg["choices"][0]["message"]["reasoning"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_tool_call(client):
|
||||
"""Tool call name+args parity."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, disagg = await _run_parity_case(
|
||||
client, messages, tools=TOOLS, tool_choice=FORCE_WEATHER_TOOL, max_tokens=1024
|
||||
)
|
||||
_assert_parity(coupled, disagg)
|
||||
|
||||
if not _tool_sig(coupled["choices"][0]):
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
assert _tool_sig(disagg["choices"][0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_reasoning_and_tool_call(client):
|
||||
"""Combined reasoning + tool call parity means the highest drift risk
|
||||
since it exercises both parser branches on the same output."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, disagg = await _run_parity_case(
|
||||
client,
|
||||
messages,
|
||||
tools=TOOLS,
|
||||
tool_choice=FORCE_WEATHER_TOOL,
|
||||
include_reasoning=True,
|
||||
max_tokens=1024,
|
||||
)
|
||||
_assert_parity(coupled, disagg)
|
||||
|
||||
c_msg = coupled["choices"][0]["message"]
|
||||
if not (c_msg.get("reasoning") and _tool_sig(coupled["choices"][0])):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
d_msg = disagg["choices"][0]["message"]
|
||||
assert d_msg["reasoning"]
|
||||
assert _tool_sig(disagg["choices"][0])
|
||||
|
||||
|
||||
def _to_generate_logprobs(openai_logprobs: dict) -> dict:
|
||||
"""Turn a coupled response's `token_id:N` logprobs into `GenerateLogProbs`.
|
||||
|
||||
A real GPU-less worker emits integer ids directly; here the coupled server
|
||||
with `return_tokens_as_token_ids=True` is the only source of the same ids,
|
||||
so the placeholders are parsed back out to build the generate-shaped
|
||||
payload that `/derender` now consumes.
|
||||
"""
|
||||
|
||||
def _id(token: str) -> int:
|
||||
assert token.startswith("token_id:"), token
|
||||
return int(token.removeprefix("token_id:"))
|
||||
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"token_id": _id(entry["token"]),
|
||||
"logprob": entry["logprob"],
|
||||
"top_logprobs": [
|
||||
{"token_id": _id(top["token"]), "logprob": top["logprob"]}
|
||||
for top in entry["top_logprobs"]
|
||||
],
|
||||
}
|
||||
for entry in openai_logprobs["content"]
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parity_logprobs(client):
|
||||
"""Integer-id logprob decoding parity vs. the coupled server's strings.
|
||||
|
||||
A real disaggregated worker only has token IDs, so it emits
|
||||
`GenerateLogProbs` with integer `token_id`s. `/derender` must decode those
|
||||
to the same token strings/bytes the coupled server produces directly.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "What is 2+2?"}]
|
||||
extra = {"logprobs": True, "top_logprobs": 3}
|
||||
|
||||
# The coupled server with placeholders is used only as a source of the
|
||||
# token ids a GPU-less worker would have had.
|
||||
placeholder = await _coupled(
|
||||
client, messages, return_tokens_as_token_ids=True, **extra
|
||||
)
|
||||
ch = placeholder["choices"][0]
|
||||
chat_request = {"model": MODEL, "messages": messages, **extra}
|
||||
disagg = await _disagg(
|
||||
client,
|
||||
ch["token_ids"],
|
||||
placeholder["usage"]["prompt_tokens"],
|
||||
ch["finish_reason"],
|
||||
chat_request,
|
||||
logprobs=_to_generate_logprobs(ch["logprobs"]),
|
||||
)
|
||||
|
||||
# The coupled server resolving the same greedy generation
|
||||
# to real token strings itself.
|
||||
resolved = await _coupled(client, messages, **extra)
|
||||
assert resolved["choices"][0]["token_ids"] == ch["token_ids"], (
|
||||
"greedy (temperature=0) generation was expected to be deterministic "
|
||||
"across the two coupled calls used to build this test's fixtures"
|
||||
)
|
||||
_assert_parity(resolved, disagg)
|
||||
|
||||
r_content = resolved["choices"][0]["logprobs"]["content"]
|
||||
d_content = disagg["choices"][0]["logprobs"]["content"]
|
||||
assert len(d_content) == len(r_content)
|
||||
for d_entry, r_entry in zip(d_content, r_content):
|
||||
assert d_entry["token"] == r_entry["token"]
|
||||
assert d_entry["bytes"] == r_entry["bytes"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _coupled_stream(
|
||||
client: httpx.AsyncClient,
|
||||
messages: list[dict],
|
||||
require_token_ids: bool = True,
|
||||
**extra,
|
||||
) -> tuple[list[dict], list[int] | None]:
|
||||
"""Stream `/v1/chat/completions` and return the choices that carry
|
||||
`token_ids` along with the prompt token IDs.
|
||||
|
||||
The leading role only chunk has no `token_ids` and no counterpart on
|
||||
`/inference/v1/generate`, so it is dropped. With `require_token_ids`
|
||||
off every choice is kept and `prompt_token_ids` may be `None`, for
|
||||
requests where the coupled server hides token ids (hidden reasoning).
|
||||
"""
|
||||
choices: list[dict] = []
|
||||
prompt_token_ids: list[int] | None = None
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"messages": messages,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"return_token_ids": True,
|
||||
"stream": True,
|
||||
**extra,
|
||||
},
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
data = line[len("data: ") :]
|
||||
if data == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(data)
|
||||
assert "error" not in chunk, chunk
|
||||
if chunk.get("prompt_token_ids") is not None:
|
||||
prompt_token_ids = chunk["prompt_token_ids"]
|
||||
choices.extend(
|
||||
ch
|
||||
for ch in chunk["choices"]
|
||||
if not require_token_ids or ch.get("token_ids") is not None
|
||||
)
|
||||
if require_token_ids:
|
||||
assert prompt_token_ids is not None
|
||||
return choices, prompt_token_ids
|
||||
|
||||
|
||||
def _tool_call_id_shape(tool_call_id: str | None) -> str | None:
|
||||
"""Drop the random suffix so IDs compare by format, e.g. `chatcmpl-tool-`.
|
||||
Deterministic IDs such as kimi_k2's `functions.<name>:<idx>` are kept
|
||||
whole."""
|
||||
if tool_call_id is None:
|
||||
return None
|
||||
return re.sub(r"[0-9a-f]{16}$", "", tool_call_id)
|
||||
|
||||
|
||||
async def _run_stream_parity_case(
|
||||
client: httpx.AsyncClient,
|
||||
messages: list[dict],
|
||||
finish_only_tail: bool = False,
|
||||
**extra,
|
||||
) -> tuple[dict, list[dict]]:
|
||||
"""Stream the coupled request, replay its chunks through the streaming
|
||||
derender endpoint and assert the assembled messages agree. Returns the
|
||||
assembled coupled message and the coupled choices.
|
||||
|
||||
Derender gets the engine's finish reason, since `"tool_calls"` is a chat
|
||||
level rewrite of `"stop"` that derender has to reapply itself. With
|
||||
`finish_only_tail` it goes on a trailing chunk with no tokens.
|
||||
"""
|
||||
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
|
||||
finish_reason = coupled_choices[-1]["finish_reason"]
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices]
|
||||
+ ([0] if finish_only_tail else []),
|
||||
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason=_engine_finish_reason(finish_reason),
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
for tc in message["tool_calls"]:
|
||||
tc["id"] = _tool_call_id_shape(tc["id"])
|
||||
assert disagg == coupled
|
||||
return coupled, coupled_choices
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming parity cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_plain(client):
|
||||
"""Plain streamed detokenization parity."""
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
await _run_stream_parity_case(client, messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_reasoning(client):
|
||||
"""Streamed reasoning/content split parity."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, _ = await _run_stream_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
|
||||
if not coupled["reasoning"]:
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_tool_call(client):
|
||||
"""Streamed tool call parity under `tool_choice="auto"`, where a tool
|
||||
call also rewrites `finish_reason` to `"tool_calls"`."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, _ = await _run_stream_parity_case(
|
||||
client, messages, tools=TOOLS, tool_choice="auto", max_tokens=512
|
||||
)
|
||||
|
||||
if not coupled["tool_calls"]:
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"chunking", ["engine_steps", "multi_token", "finish_only_tail"]
|
||||
)
|
||||
async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
"""Streamed reasoning + forced tool call parity across chunk shapes.
|
||||
|
||||
`multi_token` batches tokens per chunk with `stream_interval` the way
|
||||
speculative decoding does, so replay has to reuse the original chunk
|
||||
boundaries. `finish_only_tail` sends the finish reason on a chunk with no
|
||||
tokens, which must still flush buffered tool call arguments.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
extra: dict = {
|
||||
"tools": TOOLS,
|
||||
"tool_choice": FORCE_WEATHER_TOOL,
|
||||
"include_reasoning": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
if chunking == "multi_token":
|
||||
extra["stream_interval"] = 4
|
||||
coupled, coupled_choices = await _run_stream_parity_case(
|
||||
client,
|
||||
messages,
|
||||
finish_only_tail=chunking == "finish_only_tail",
|
||||
**extra,
|
||||
)
|
||||
|
||||
if chunking == "multi_token":
|
||||
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
|
||||
if not (coupled["reasoning"] and coupled["tool_calls"]):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming logprobs on the parser path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
return [(e["token"], e.get("bytes")) for e in lp["content"]]
|
||||
|
||||
|
||||
def _sampled_generate_logprobs(choice: dict) -> dict | None:
|
||||
"""`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
|
||||
and per-token logprob values, without `top_logprobs`.
|
||||
|
||||
The coupled stream's `token_ids` are the integer ids a GPU-less worker
|
||||
would have sent, so one stream is both the derender input and the string
|
||||
reference. A second greedy run is not needed, and greedy decoding is not
|
||||
bit-reproducible across calls on every platform anyway.
|
||||
"""
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
content = lp["content"]
|
||||
assert len(content) == len(choice["token_ids"]), choice
|
||||
return {
|
||||
"content": [
|
||||
{"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
|
||||
for tid, entry in zip(choice["token_ids"], content)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def _run_stream_logprob_parity_case(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[dict, list[dict], list[dict]]:
|
||||
"""Stream with logprobs through the coupled server, replay the same
|
||||
chunks through the streaming derender endpoint and compare per chunk.
|
||||
Returns the assembled coupled message, the coupled choices and the
|
||||
derender choices.
|
||||
"""
|
||||
extra = {"logprobs": True, **extra}
|
||||
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
|
||||
finish_reason = coupled_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices],
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
for tc in message["tool_calls"]:
|
||||
tc["id"] = _tool_call_id_shape(tc["id"])
|
||||
assert disagg == coupled
|
||||
assert len(derender_choices) == len(coupled_choices)
|
||||
for ref, der in zip(coupled_choices, derender_choices):
|
||||
assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
|
||||
return coupled, coupled_choices, derender_choices
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_reasoning(client):
|
||||
"""Per-chunk logprobs on the parser path match the coupled stream across
|
||||
the reasoning -> content transition."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
if not coupled["reasoning"]:
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
assert any(
|
||||
ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_tool_call(client):
|
||||
"""Chunks the parser swallows (tool call markup) still carry their
|
||||
logprobs, on both sides, so no token's logprobs are lost."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client,
|
||||
messages,
|
||||
tools=TOOLS,
|
||||
tool_choice=FORCE_WEATHER_TOOL,
|
||||
include_reasoning=True,
|
||||
max_tokens=512,
|
||||
)
|
||||
if not coupled["tool_calls"]:
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
swallowed = [
|
||||
ch
|
||||
for ch in derender_choices
|
||||
if ch.get("logprobs")
|
||||
and not ch["delta"].get("content")
|
||||
and not ch["delta"].get("reasoning")
|
||||
and not ch["delta"].get("tool_calls")
|
||||
]
|
||||
assert swallowed, "expected at least one buffered-markup chunk with logprobs"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_hidden_reasoning(client):
|
||||
"""With `include_reasoning=False` and a parser configured, neither side
|
||||
returns logprobs on any chunk; derender still emits the content."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
extra = {"logprobs": True, "max_tokens": 256}
|
||||
# Token ids come from a shown-reasoning run: the coupled server hides
|
||||
# them along with the logprobs when reasoning is hidden.
|
||||
shown_choices, prompt_token_ids = await _coupled_stream(
|
||||
client, messages, include_reasoning=True, **extra
|
||||
)
|
||||
hidden_choices, _ = await _coupled_stream(
|
||||
client, messages, require_token_ids=False, include_reasoning=False, **extra
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in hidden_choices)
|
||||
assert assemble_stream(hidden_choices)["reasoning"] is None
|
||||
|
||||
finish_reason = shown_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in shown_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in shown_choices],
|
||||
{"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in derender_choices)
|
||||
shown = assemble_stream(shown_choices)
|
||||
assert disagg["reasoning"] is None
|
||||
assert disagg["content"] == shown["content"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline text level: generate(output_mode="text") == derender(generate(...))
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TEXT_MESSAGES = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
|
||||
|
||||
async def _render_token_ids(
|
||||
client: httpx.AsyncClient, messages: list[dict]
|
||||
) -> list[int]:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["token_ids"]
|
||||
|
||||
|
||||
def _generate_payload(
|
||||
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
|
||||
) -> dict:
|
||||
return {
|
||||
"model": MODEL,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
|
||||
"output_mode": output_mode,
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
|
||||
async def _generate(
|
||||
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
|
||||
) -> dict:
|
||||
resp = await client.post(
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, output_mode, **sampling),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == output_mode
|
||||
return data
|
||||
|
||||
|
||||
async def _generate_text_stream(
|
||||
client: httpx.AsyncClient, token_ids: list[int], **sampling
|
||||
) -> list[dict]:
|
||||
"""Stream generate at the text level and return the choice of every chunk
|
||||
that has one."""
|
||||
choices: list[dict] = []
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, "text", stream=True, **sampling),
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
assert "error" not in chunk, chunk
|
||||
assert chunk["output_mode"] == "text"
|
||||
choices.extend(chunk["choices"])
|
||||
return choices
|
||||
|
||||
|
||||
async def _derender_completion_text(
|
||||
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
|
||||
) -> str:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"generate_responses": [
|
||||
{
|
||||
"request_id": "text-parity",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": output_ids,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
async def _derender_completion_stream_text(
|
||||
client: httpx.AsyncClient, choices: list[dict]
|
||||
) -> str:
|
||||
"""Replay each generate chunk's token IDs through the streaming
|
||||
completions derender endpoint with the same chunk boundaries."""
|
||||
state = None
|
||||
text = ""
|
||||
for choice in choices:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": MODEL,
|
||||
"generate_chunk": {
|
||||
"request_id": "text-parity-stream",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": choice.get("token_ids") or [],
|
||||
"finish_reason": choice.get("finish_reason"),
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
state = data["stream_state"]
|
||||
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
|
||||
return text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_batch(client):
|
||||
"""Inline text matches /v1/completions/derender on the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
inline = (await _generate(client, token_ids, "text"))["choices"][0]
|
||||
|
||||
derendered = await _derender_completion_text(
|
||||
client, inline["token_ids"], inline["finish_reason"]
|
||||
)
|
||||
|
||||
assert inline["text"]
|
||||
assert inline["text"] == derendered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_stream(client):
|
||||
"""Streamed inline text matches both the streaming and the non-streaming
|
||||
derender of the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
choices = await _generate_text_stream(client, token_ids)
|
||||
inline = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
|
||||
assert finish_reason is not None
|
||||
assert inline == await _derender_completion_stream_text(client, choices)
|
||||
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_logprobs(client):
|
||||
"""Inline logprob tokens and bytes match /derender resolving the
|
||||
token_id:N placeholders of the tokens level."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
sampling = {"logprobs": 3}
|
||||
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
|
||||
"choices"
|
||||
][0]
|
||||
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
|
||||
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
|
||||
"greedy (temperature=0) generation was expected to be deterministic "
|
||||
"across the two generate calls"
|
||||
)
|
||||
|
||||
disagg = await _disagg(
|
||||
client,
|
||||
tokens_choice["token_ids"],
|
||||
len(token_ids),
|
||||
tokens_choice["finish_reason"],
|
||||
{"model": MODEL, "messages": TEXT_MESSAGES},
|
||||
logprobs=tokens_choice["logprobs"],
|
||||
)
|
||||
|
||||
inline_content = text_choice["logprobs"]["content"]
|
||||
derender_content = disagg["choices"][0]["logprobs"]["content"]
|
||||
assert len(inline_content) == len(derender_content)
|
||||
for inline_entry, derender_entry in zip(inline_content, derender_content):
|
||||
assert inline_entry["token"] == derender_entry["token"]
|
||||
assert inline_entry["bytes"] == derender_entry["bytes"]
|
||||
assert [
|
||||
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
|
||||
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_text_parity_stop_string(client, stream):
|
||||
"""Inline text follows the engine's stop handling, like /v1/completions.
|
||||
|
||||
token_ids keep every generated token, so /derender brings the matched
|
||||
stop string back. Its text must start with the inline text and only
|
||||
differ after it.
|
||||
"""
|
||||
stop = "\n"
|
||||
token_ids = await _render_token_ids(
|
||||
client,
|
||||
[{"role": "user", "content": "Write two short lines about the sea."}],
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
"/v1/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"prompt": token_ids,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"stop": [stop],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
coupled = resp.json()["choices"][0]
|
||||
|
||||
if stream:
|
||||
choices = await _generate_text_stream(client, token_ids, stop=[stop])
|
||||
inline_text = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
else:
|
||||
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
|
||||
inline_text = inline["text"]
|
||||
output_ids = inline["token_ids"]
|
||||
finish_reason = inline["finish_reason"]
|
||||
|
||||
assert inline_text == coupled["text"]
|
||||
assert finish_reason == coupled["finish_reason"]
|
||||
assert stop not in inline_text
|
||||
|
||||
derendered = await _derender_completion_text(client, output_ids, finish_reason)
|
||||
assert derendered.startswith(inline_text)
|
||||
if coupled["stop_reason"] != stop:
|
||||
pytest.skip("Model did not emit the stop string")
|
||||
assert derendered[len(inline_text) :].startswith(stop)
|
||||
@@ -1,118 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers shared by the streaming chat derender tests."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def assemble_stream(choices: list[dict]) -> dict:
|
||||
"""Fold streamed chat choices into one message.
|
||||
|
||||
A tool call index must keep the first ID it was given. On the derender
|
||||
side that is what pinning the IDs in `stream_state` guarantees.
|
||||
"""
|
||||
content = ""
|
||||
reasoning = ""
|
||||
tool_calls: list[dict] = []
|
||||
finish_reason = None
|
||||
for choice in choices:
|
||||
delta = choice["delta"]
|
||||
content += delta.get("content") or ""
|
||||
reasoning += delta.get("reasoning") or ""
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
idx = tc["index"]
|
||||
while len(tool_calls) <= idx:
|
||||
tool_calls.append({"id": None, "name": None, "arguments": ""})
|
||||
if tc.get("id"):
|
||||
assert tool_calls[idx]["id"] in (None, tc["id"]), (
|
||||
f"tool call {idx} changed ID mid stream"
|
||||
)
|
||||
tool_calls[idx]["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
tool_calls[idx]["name"] = fn["name"]
|
||||
tool_calls[idx]["arguments"] += fn.get("arguments") or ""
|
||||
finish_reason = choice.get("finish_reason") or finish_reason
|
||||
return {
|
||||
"content": content or None,
|
||||
"reasoning": reasoning or None,
|
||||
"tool_calls": tool_calls,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
|
||||
|
||||
async def stream_chat_derender(
|
||||
client: httpx.AsyncClient,
|
||||
output_ids: list[int],
|
||||
chunk_sizes: list[int],
|
||||
chat_request: dict,
|
||||
prompt_tokens: int,
|
||||
prompt_token_ids: list[int],
|
||||
finish_reason: str = "stop",
|
||||
on_chunk: Callable[[list[dict]], None] | None = None,
|
||||
logprobs: list[dict | None] | None = None,
|
||||
raw_choices: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Feed `output_ids` through the streaming chat derender endpoint in
|
||||
the given `chunk_sizes`, threading `stream_state` across calls and
|
||||
return the assembled message.
|
||||
|
||||
`finish_reason` goes on the last chunk, so a trailing size of 0 sends it
|
||||
on a chunk with no tokens.
|
||||
|
||||
If `on_chunk` is given, it is called after every chunk with the
|
||||
`tool_calls` assembled so far, letting callers assert properties of the
|
||||
intermediate deltas (e.g. monotonic argument growth) rather than only
|
||||
the final assembled result.
|
||||
|
||||
`logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
|
||||
and `raw_choices`, if given, collects every derendered choice so callers
|
||||
can compare per-chunk fields such as `logprobs`.
|
||||
"""
|
||||
if logprobs is not None:
|
||||
assert len(logprobs) == len(chunk_sizes)
|
||||
state = None
|
||||
choices: list[dict] = []
|
||||
pos = 0
|
||||
for i, size in enumerate(chunk_sizes):
|
||||
tids = output_ids[pos : pos + size]
|
||||
pos += size
|
||||
is_last = i == len(chunk_sizes) - 1
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": chat_request["model"],
|
||||
"generate_chunk": {
|
||||
"request_id": "stream-test",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": tids,
|
||||
"finish_reason": finish_reason if is_last else None,
|
||||
"logprobs": logprobs[i] if logprobs is not None else None,
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"prompt_token_ids": prompt_token_ids,
|
||||
"chat_request": chat_request,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
if chat_request.get("return_token_ids") and chat_request.get(
|
||||
"include_reasoning", True
|
||||
):
|
||||
assert data["chunk"]["choices"][0]["token_ids"] == tids
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if raw_choices is not None:
|
||||
raw_choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
on_chunk(assemble_stream(choices)["tool_calls"])
|
||||
|
||||
return assemble_stream(choices)
|
||||
@@ -1,43 +0,0 @@
|
||||
# Scale-out EC connector e2e tests
|
||||
|
||||
End-to-end tests for the disaggregated multimodal (encoder-cache) flow over
|
||||
the scale-out endpoints, in the bash-script style of
|
||||
`tests/v1/kv_connector/nixl_integration`.
|
||||
|
||||
## Topology
|
||||
|
||||
```text
|
||||
render (GPU-less) encode (EC producer)
|
||||
/v1/chat/completions/render /inference/v1/generate (kwargs_data)
|
||||
/v1/chat/completions/derender |
|
||||
prefill (EC consumer)
|
||||
/inference/v1/generate (mm_metadata + ec_transfer_params)
|
||||
```
|
||||
|
||||
- The **render** server (`vllm launch render`) preprocesses the chat request
|
||||
into token ids and multimodal features (`kwargs_data` + `mm_metadata`).
|
||||
- The **encode** instance (`ec_role: ec_producer`, encode-only) runs the
|
||||
vision encoder on `kwargs_data` and publishes embeddings through the
|
||||
`ECExampleConnector` shared storage. Its response carries
|
||||
`ec_transfer_params`.
|
||||
- The **prefill** instance (`ec_role: ec_consumer`) receives metadata-only
|
||||
features (`mm_metadata`, no `kwargs_data`) plus `ec_transfer_params`, loads
|
||||
the embeddings through the EC connector, and generates output token ids.
|
||||
- The **derender** endpoint turns the output token ids back into a chat
|
||||
completion response.
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
|
||||
```
|
||||
|
||||
The script first collects baseline outputs from a single `vllm serve`
|
||||
instance, then runs the scale-out topology and compares the derendered
|
||||
outputs for exact equality. It also checks that metadata-only features are
|
||||
rejected when `ec_transfer_params` is missing.
|
||||
|
||||
Defaults: `Qwen/Qwen3-VL-2B-Instruct`, encode on GPU 0, prefill on GPU 1.
|
||||
Override via environment variables (`MODEL`, `GPU_E`, `GPU_PD`,
|
||||
`EC_SHARED_STORAGE_PATH`, `PYTHON=.venv/bin/python`, ...); see the header of
|
||||
the script for the full list.
|
||||
@@ -1,277 +0,0 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Scale-out EC connector E2E test
|
||||
#
|
||||
# Tests the disaggregated multimodal flow over the scale-out endpoints,
|
||||
# following the bash-script style of tests/v1/kv_connector/nixl_integration:
|
||||
#
|
||||
# 1. Baseline: a single vLLM instance serving /v1/chat/completions
|
||||
# 2. Scale-out:
|
||||
# render (GPU-less, `vllm launch render`)
|
||||
# encode (EC producer, encode-only)
|
||||
# prefill (EC consumer)
|
||||
#
|
||||
# The Python client renders each multimodal request once, sends the full
|
||||
# kwargs_data to the encode instance, then sends metadata-only features
|
||||
# (mm_metadata, no kwargs_data) plus ec_transfer_params to the prefill
|
||||
# instance, and derenders the output tokens. Outputs are compared against
|
||||
# the baseline for exact equality.
|
||||
#
|
||||
# Usage:
|
||||
# bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
|
||||
#
|
||||
# Requires 2 GPUs by default (encode on GPU 0, prefill on GPU 1); the
|
||||
# baseline reuses GPU 0 after the encode instance is stopped.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
|
||||
# Model and engine configuration
|
||||
MODEL="${MODEL:-Qwen/Qwen3-VL-2B-Instruct}"
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-8192}"
|
||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
|
||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||||
|
||||
# GPU configuration. encode + prefill run concurrently on separate GPUs;
|
||||
# the baseline reuses GPU_E after the scale-out instances are stopped.
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_PD="${GPU_PD:-1}"
|
||||
GPU_SINGLE="${GPU_SINGLE:-$GPU_E}"
|
||||
|
||||
# Ports
|
||||
RENDER_PORT="${RENDER_PORT:-19600}"
|
||||
ENCODE_PORT="${ENCODE_PORT:-19601}"
|
||||
PREFILL_PORT="${PREFILL_PORT:-19602}"
|
||||
BASELINE_PORT="${BASELINE_PORT:-19603}"
|
||||
|
||||
# Storage path for the ECExampleConnector encoder cache
|
||||
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_scale_out_test}"
|
||||
|
||||
# Python interpreter (override to .venv/bin/python for local runs)
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
# Output files and logs
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_scale_out_ec_baseline.txt}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
|
||||
BASELINE_LOG="${LOG_PATH}/scale_out_ec_baseline.log"
|
||||
RENDER_LOG="${LOG_PATH}/scale_out_ec_render.log"
|
||||
ENCODE_LOG="${LOG_PATH}/scale_out_ec_encode.log"
|
||||
PREFILL_LOG="${LOG_PATH}/scale_out_ec_prefill.log"
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local pid=$2
|
||||
local name=$3
|
||||
local deadline=$((SECONDS + TIMEOUT_SECONDS))
|
||||
|
||||
while ((SECONDS < deadline)); do
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
echo "$name instance exited before becoming healthy"
|
||||
return 1
|
||||
fi
|
||||
if curl --max-time 2 -fsS "http://localhost:${port}/health" \
|
||||
> /dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Timed out waiting for $name instance on port $port"
|
||||
return 1
|
||||
}
|
||||
|
||||
dump_log() {
|
||||
local name=$1
|
||||
local log=$2
|
||||
|
||||
echo "--- $name log (last 200 lines)"
|
||||
if [[ -f "$log" ]]; then
|
||||
tail -n 200 "$log" || true
|
||||
else
|
||||
echo "Log file not found: $log"
|
||||
fi
|
||||
}
|
||||
|
||||
dump_scale_out_logs() {
|
||||
dump_log "render" "$RENDER_LOG"
|
||||
dump_log "encode" "$ENCODE_LOG"
|
||||
dump_log "prefill" "$PREFILL_LOG"
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM instances..."
|
||||
pkill -TERM -f "vllm serve" || true
|
||||
pkill -TERM -f "vllm launch render" || true
|
||||
sleep 3
|
||||
pkill -9 -f "vllm serve" || true
|
||||
pkill -9 -f "vllm launch render" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
# Step 1: baseline single instance
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "Running BASELINE (single instance)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
echo "Starting baseline instance on GPU $GPU_SINGLE, port $BASELINE_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
|
||||
--port "$BASELINE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
> "$BASELINE_LOG" 2>&1 &
|
||||
local BASELINE_PID=$!
|
||||
|
||||
echo "Waiting for baseline instance to start..."
|
||||
if ! wait_for_server "$BASELINE_PORT" "$BASELINE_PID" "baseline"; then
|
||||
dump_log "baseline" "$BASELINE_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
|
||||
--mode baseline \
|
||||
--service_url "http://localhost:$BASELINE_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--baseline_file "$BASELINE_FILE"; then
|
||||
dump_log "baseline" "$BASELINE_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Stopping baseline instance..."
|
||||
kill "$BASELINE_PID" 2>/dev/null || true
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Step 2: render + encode (EC producer) + prefill (EC consumer)
|
||||
run_scale_out_ec() {
|
||||
echo "================================"
|
||||
echo "Running SCALE-OUT EC (render + encode + prefill)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
mkdir -p "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
declare -a PIDS=()
|
||||
|
||||
# GPU-less render/derender server
|
||||
echo "Starting render server on port $RENDER_PORT"
|
||||
vllm launch render "$MODEL" \
|
||||
--port "$RENDER_PORT" \
|
||||
> "$RENDER_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
# Encode-only EC producer instance. It runs the vision encoder and
|
||||
# publishes embeddings through the ECExampleConnector shared storage.
|
||||
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_E" \
|
||||
vllm serve "$MODEL" \
|
||||
--enable-scale-out \
|
||||
--port "$ENCODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$ENCODE_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
# Prefill/decode EC consumer instance. It receives metadata-only
|
||||
# features plus ec_transfer_params and loads the embeddings that the
|
||||
# encode instance published.
|
||||
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_PD" \
|
||||
vllm serve "$MODEL" \
|
||||
--enable-scale-out \
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$PREFILL_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
echo "Waiting for render instance..."
|
||||
if ! wait_for_server "$RENDER_PORT" "${PIDS[0]}" "render"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
echo "Waiting for encode instance..."
|
||||
if ! wait_for_server "$ENCODE_PORT" "${PIDS[1]}" "encode"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
echo "Waiting for prefill instance..."
|
||||
if ! wait_for_server "$PREFILL_PORT" "${PIDS[2]}" "prefill"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "All scale-out EC services are up!"
|
||||
|
||||
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
|
||||
--mode disagg \
|
||||
--render_url "http://localhost:$RENDER_PORT" \
|
||||
--encode_url "http://localhost:$ENCODE_PORT" \
|
||||
--prefill_url "http://localhost:$PREFILL_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--baseline_file "$BASELINE_FILE"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Stopping scale-out EC instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Main execution
|
||||
echo "================================"
|
||||
echo "Scale-out EC E2E Test Suite"
|
||||
echo "Model: $MODEL"
|
||||
echo "================================"
|
||||
|
||||
run_baseline
|
||||
run_scale_out_ec
|
||||
|
||||
rm -f "$BASELINE_FILE"
|
||||
|
||||
echo "================================"
|
||||
echo "All scale-out EC e2e tests finished!"
|
||||
echo "================================"
|
||||
@@ -1,276 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E test for the scale-out EC connector flow (render -> encode -> prefill).
|
||||
|
||||
Mimics the bash-script style of ``tests/v1/kv_connector/nixl_integration``:
|
||||
the instances are started by ``run_scale_out_ec_e2e_test.sh`` and this file is
|
||||
the HTTP client that drives the disaggregated request path and compares its
|
||||
output against a single-instance baseline.
|
||||
|
||||
Topology under test (started by the bash script):
|
||||
|
||||
render (GPU-less) -- /v1/chat/completions/render, /derender
|
||||
encode (EC producer) -- /inference/v1/generate
|
||||
prefill (EC consumer) -- /inference/v1/generate
|
||||
|
||||
The client renders each multimodal chat request once, sends the full
|
||||
``kwargs_data`` to the encode instance, then sends metadata-only features
|
||||
(``mm_metadata`` without ``kwargs_data``) plus the encode response's
|
||||
``ec_transfer_params`` to the prefill instance, and finally derenders the
|
||||
output token ids. The derendered text must match the baseline exactly.
|
||||
|
||||
Usage (from the bash script, or manually):
|
||||
# Baseline mode (saves outputs):
|
||||
python test_scale_out_ec_e2e.py \
|
||||
--mode baseline \
|
||||
--service_url http://localhost:19603 \
|
||||
--model_name Qwen/Qwen3-VL-2B-Instruct \
|
||||
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
|
||||
|
||||
# Disagg mode (compares outputs):
|
||||
python test_scale_out_ec_e2e.py \
|
||||
--mode disagg \
|
||||
--render_url http://localhost:19600 \
|
||||
--encode_url http://localhost:19601 \
|
||||
--prefill_url http://localhost:19602 \
|
||||
--model_name Qwen/Qwen3-VL-2B-Instruct \
|
||||
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MAX_OUTPUT_LEN = 128
|
||||
REQUEST_TIMEOUT = 300
|
||||
|
||||
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
|
||||
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
|
||||
image_1_url = encode_image_url(image_1)
|
||||
image_2_url = encode_image_url(image_2)
|
||||
|
||||
# Each prompt is a chat completion request body; the same body is sent to the
|
||||
# baseline instance, to /render, and to /derender (as chat_request).
|
||||
SAMPLE_PROMPTS: list[dict] = [
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_1_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "single image",
|
||||
},
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_2_url}},
|
||||
{"type": "image_url", "image_url": {"url": image_1_url}},
|
||||
{"type": "text", "text": "Describe these 2 images in detail."},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "two images",
|
||||
},
|
||||
{
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"description": "text only",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def chat_request(prompt: dict) -> dict:
|
||||
return {
|
||||
"model": None, # filled by the caller
|
||||
"messages": prompt["messages"],
|
||||
"max_tokens": MAX_OUTPUT_LEN,
|
||||
"temperature": 0.0,
|
||||
"seed": 42,
|
||||
}
|
||||
|
||||
|
||||
def post_json(url: str, payload: dict) -> requests.Response:
|
||||
response = requests.post(url, json=payload, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"POST {url} failed with {response.status_code}: {response.text}"
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def run_baseline(args) -> None:
|
||||
outputs = []
|
||||
for prompt in SAMPLE_PROMPTS:
|
||||
request = chat_request(prompt) | {"model": args.model_name}
|
||||
response = post_json(f"{args.service_url}/v1/chat/completions", request).json()
|
||||
outputs.append(response["choices"][0]["message"]["content"])
|
||||
print(f"[baseline] {prompt['description']}: {outputs[-1][:80]!r}")
|
||||
|
||||
with open(args.baseline_file, "w") as f:
|
||||
json.dump(outputs, f)
|
||||
print(f"Baseline saved to {args.baseline_file}")
|
||||
|
||||
|
||||
def _assert_mm_features(features: dict) -> None:
|
||||
kwargs_data = features.get("kwargs_data") or {}
|
||||
mm_metadata = features.get("mm_metadata")
|
||||
assert any(item is not None for items in kwargs_data.values() for item in items), (
|
||||
f"render response has no kwargs_data items: {features}"
|
||||
)
|
||||
assert mm_metadata, "render response is missing mm_metadata"
|
||||
for modality, hashes in features["mm_hashes"].items():
|
||||
assert modality in mm_metadata, f"mm_metadata missing modality {modality}"
|
||||
assert len(mm_metadata[modality]) == len(hashes)
|
||||
|
||||
|
||||
def run_disagg(args) -> None:
|
||||
with open(args.baseline_file) as f:
|
||||
baseline = json.load(f)
|
||||
|
||||
failures = []
|
||||
for prompt, expected in zip(SAMPLE_PROMPTS, baseline, strict=True):
|
||||
request = chat_request(prompt) | {"model": args.model_name}
|
||||
got = run_disagg_one(args, request)
|
||||
if got != expected:
|
||||
failures.append(
|
||||
f"{prompt['description']}:\n"
|
||||
f" baseline: {expected!r}\n disagg: {got!r}"
|
||||
)
|
||||
else:
|
||||
print(f"[disagg] {prompt['description']}: matched baseline")
|
||||
|
||||
if failures:
|
||||
print("Output mismatch:")
|
||||
for failure in failures:
|
||||
print(failure)
|
||||
sys.exit(1)
|
||||
print("All scale-out EC outputs matched the baseline")
|
||||
|
||||
|
||||
def run_disagg_one(args, request: dict) -> str:
|
||||
# 1. Render: chat request -> token ids + multimodal features (GPU-less).
|
||||
rendered = post_json(
|
||||
f"{args.render_url}/v1/chat/completions/render", request
|
||||
).json()
|
||||
token_ids = rendered["token_ids"]
|
||||
features = rendered.get("features")
|
||||
request_id = rendered["request_id"]
|
||||
|
||||
# 2. Encode: full kwargs_data so the vision encoder can run. The
|
||||
# response carries ec_transfer_params for the prefill instance.
|
||||
ec_transfer_params = None
|
||||
if features is not None:
|
||||
_assert_mm_features(features)
|
||||
encode_payload = {
|
||||
"request_id": request_id,
|
||||
"token_ids": token_ids,
|
||||
"features": {
|
||||
"mm_hashes": features["mm_hashes"],
|
||||
"mm_placeholders": features["mm_placeholders"],
|
||||
"kwargs_data": features["kwargs_data"],
|
||||
},
|
||||
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
|
||||
}
|
||||
encode_response = post_json(
|
||||
f"{args.encode_url}/inference/v1/generate", encode_payload
|
||||
).json()
|
||||
ec_transfer_params = encode_response.get("ec_transfer_params")
|
||||
assert ec_transfer_params, (
|
||||
f"encode response has no ec_transfer_params: {encode_response}"
|
||||
)
|
||||
|
||||
# 3. Prefill: metadata-only features. kwargs_data is dropped; the
|
||||
# embeddings arrive through the EC connector.
|
||||
if features is not None:
|
||||
prefill_features = {
|
||||
"mm_hashes": features["mm_hashes"],
|
||||
"mm_placeholders": features["mm_placeholders"],
|
||||
"mm_metadata": features["mm_metadata"],
|
||||
}
|
||||
_check_metadata_only_rejected_without_ec(args, token_ids, prefill_features)
|
||||
else:
|
||||
prefill_features = None
|
||||
|
||||
prefill_payload = {
|
||||
"request_id": request_id,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": MAX_OUTPUT_LEN, "temperature": 0.0},
|
||||
}
|
||||
if prefill_features is not None:
|
||||
prefill_payload["features"] = prefill_features
|
||||
prefill_payload["ec_transfer_params"] = ec_transfer_params
|
||||
prefill_response = post_json(
|
||||
f"{args.prefill_url}/inference/v1/generate", prefill_payload
|
||||
).json()
|
||||
output_token_ids = prefill_response["choices"][0]["token_ids"]
|
||||
assert output_token_ids, f"prefill returned no token ids: {prefill_response}"
|
||||
|
||||
# 4. Derender: token ids -> chat completion response (GPU-less).
|
||||
derendered = post_json(
|
||||
f"{args.render_url}/v1/chat/completions/derender",
|
||||
{
|
||||
"model": args.model_name,
|
||||
"generate_response": prefill_response,
|
||||
"prompt_tokens": len(token_ids),
|
||||
"chat_request": request,
|
||||
},
|
||||
).json()
|
||||
return derendered["choices"][0]["message"]["content"]
|
||||
|
||||
|
||||
def _check_metadata_only_rejected_without_ec(
|
||||
args, token_ids: list[int], prefill_features: dict
|
||||
) -> None:
|
||||
"""mm_metadata without kwargs_data must be rejected without EC params."""
|
||||
payload = {
|
||||
"token_ids": token_ids,
|
||||
"features": prefill_features,
|
||||
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
|
||||
}
|
||||
response = requests.post(
|
||||
f"{args.prefill_url}/inference/v1/generate",
|
||||
json=payload,
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
assert response.status_code in (400, 422), (
|
||||
"metadata-only features without ec_transfer_params should be rejected, "
|
||||
f"got {response.status_code}: {response.text}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mode", choices=["baseline", "disagg"], required=True)
|
||||
parser.add_argument("--service_url", help="baseline single-instance URL")
|
||||
parser.add_argument("--render_url", help="GPU-less render server URL")
|
||||
parser.add_argument("--encode_url", help="EC producer (encode) URL")
|
||||
parser.add_argument("--prefill_url", help="EC consumer (prefill) URL")
|
||||
parser.add_argument("--model_name", required=True)
|
||||
parser.add_argument("--baseline_file", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
start = time.time()
|
||||
if args.mode == "baseline":
|
||||
assert args.service_url, "--service_url is required in baseline mode"
|
||||
run_baseline(args)
|
||||
else:
|
||||
assert args.render_url, "--render_url is required in disagg mode"
|
||||
assert args.encode_url, "--encode_url is required in disagg mode"
|
||||
assert args.prefill_url, "--prefill_url is required in disagg mode"
|
||||
run_disagg(args)
|
||||
print(f"Mode {args.mode} finished in {time.time() - start:.1f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E tests for render endpoints via `vllm launch` (GPU-less serving)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = []
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args, max_wait_seconds=120) as srv:
|
||||
yield srv
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# -- Chat Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Response should be a GenerateRequest dict
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in data["token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_multi_turn(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
|
||||
# -- Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Once upon a time",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) > 0
|
||||
|
||||
first_prompt = data[0]
|
||||
assert "token_ids" in first_prompt
|
||||
assert "sampling_params" in first_prompt
|
||||
assert "model" in first_prompt
|
||||
assert "request_id" in first_prompt
|
||||
assert isinstance(first_prompt["token_ids"], list)
|
||||
assert len(first_prompt["token_ids"]) > 0
|
||||
assert first_prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
for prompt in data:
|
||||
assert "token_ids" in prompt
|
||||
assert "sampling_params" in prompt
|
||||
assert "model" in prompt
|
||||
assert "request_id" in prompt
|
||||
assert len(prompt["token_ids"]) > 0
|
||||
assert prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_invalid_model(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": "nonexistent-model",
|
||||
"prompt": "Hello",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "error" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_is_fast(client):
|
||||
"""Render should complete quickly since there is no inference."""
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Tell me a very long story about " * 10,
|
||||
},
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert response.status_code == 200
|
||||
assert elapsed < 2.0
|
||||
|
||||
|
||||
# -- Health & Models --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint(client):
|
||||
response = await client.get("/health")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_endpoint(client):
|
||||
response = await client.get("/v1/models")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "data" in data
|
||||
model_ids = [m["id"] for m in data["data"]]
|
||||
assert MODEL_NAME in model_ids
|
||||
@@ -1,882 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for the /render endpoints that expose prompt preprocessing."""
|
||||
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.render.api_router import router
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
def _build_serving_render() -> ServingRender:
|
||||
serving = ServingRender.__new__(ServingRender)
|
||||
serving.model_config = SimpleNamespace(
|
||||
max_model_len=100,
|
||||
is_encoder_decoder=False,
|
||||
)
|
||||
serving.default_sampling_params = {}
|
||||
serving.override_max_tokens = None
|
||||
serving.tool_server = MagicMock()
|
||||
serving.online_renderer = MagicMock()
|
||||
serving.online_renderer.create_error_response = (
|
||||
OnlineRenderer.create_error_response.__get__(serving.online_renderer)
|
||||
)
|
||||
serving.online_renderer.validate_chat_template = (
|
||||
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
|
||||
)
|
||||
serving.online_renderer.trust_request_chat_template = True
|
||||
serving.online_renderer.parser = None
|
||||
serving._check_model = AsyncMock(return_value=None)
|
||||
return serving
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_completion_rejects_beam_search():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [7, 8, 9]}]
|
||||
)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
use_beam_search=True,
|
||||
n=2,
|
||||
length_penalty=0.7,
|
||||
)
|
||||
|
||||
response = await serving.render_completion_request(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert (
|
||||
response.error.message == "Beam search is not supported by the render endpoint"
|
||||
)
|
||||
serving.online_renderer.render_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_responses_render_route_is_registered():
|
||||
assert any(route.path == "/v1/responses/render" for route in router.routes)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_responses_render_route_documents_not_implemented():
|
||||
route = next(
|
||||
route for route in router.routes if route.path == "/v1/responses/render"
|
||||
)
|
||||
|
||||
assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_returns_generate_request_without_stored_state():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
engine_input={"prompt_token_ids": [7, 8, 9]},
|
||||
)
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
max_output_tokens=12,
|
||||
stream=True,
|
||||
cache_salt="request-salt",
|
||||
priority=3,
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
ec_transfer_params={"remote": True},
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert response.token_ids == [7, 8, 9]
|
||||
assert response.request_id == request.request_id
|
||||
assert response.sampling_params.max_tokens == 12
|
||||
assert response.model == MODEL_NAME
|
||||
assert response.stream is True
|
||||
assert response.cache_salt == "request-salt"
|
||||
assert response.priority == 3
|
||||
assert response.kv_transfer_params == {"do_remote_prefill": True}
|
||||
assert response.ec_transfer_params == {"remote": True}
|
||||
assert response.sampling_params.extra_args is None
|
||||
serving.online_renderer.render_responses.assert_awaited_once_with(
|
||||
request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=serving.tool_server,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_chat_hoists_transfer_params_out_of_extra_args():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_chat = AsyncMock(
|
||||
return_value=([], [{"prompt_token_ids": [7, 8, 9]}])
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
ec_transfer_params={"remote": True},
|
||||
vllm_xargs={"foo": 1},
|
||||
)
|
||||
|
||||
response = await serving.render_chat_request(request)
|
||||
|
||||
assert response.kv_transfer_params == {"do_remote_prefill": True}
|
||||
assert response.ec_transfer_params == {"remote": True}
|
||||
assert response.sampling_params.extra_args == {"foo": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_completion_hoists_transfer_params_for_every_prompt():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_completion = AsyncMock(
|
||||
return_value=[
|
||||
{"prompt_token_ids": [7, 8, 9]},
|
||||
{"prompt_token_ids": [4, 5]},
|
||||
]
|
||||
)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt=["a", "b"],
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
ec_transfer_params={"remote": True},
|
||||
)
|
||||
|
||||
responses = await serving.render_completion_request(request)
|
||||
|
||||
assert len(responses) == 2
|
||||
for response in responses:
|
||||
assert response.kv_transfer_params == {"do_remote_prefill": True}
|
||||
assert response.ec_transfer_params == {"remote": True}
|
||||
assert response.sampling_params.extra_args is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_rejects_previous_response_id():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock()
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
previous_response_id="resp_previous",
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert response.error.param == "previous_response_id"
|
||||
serving.online_renderer.render_responses.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_rejects_empty_token_ids():
|
||||
serving = _build_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
messages=[],
|
||||
engine_input={"prompt_token_ids": []},
|
||||
)
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(
|
||||
ResponsesRequest(model=MODEL_NAME, input="Test prompt")
|
||||
)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.message == "No token_ids rendered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_carries_reasoning_parser_kwargs():
|
||||
"""/v1/responses gives the engine's reasoning parser the effective template
|
||||
kwargs, so the rendered request must carry them to generate."""
|
||||
serving = _build_serving_render()
|
||||
online = serving.online_renderer
|
||||
online.parser = MagicMock(reasoning_parser_cls=object)
|
||||
online.chat_template = None
|
||||
online.chat_template_content_format = "auto"
|
||||
online.default_chat_template_kwargs = {"enable_thinking": True}
|
||||
online.effective_chat_template_kwargs = (
|
||||
OnlineRenderer.effective_chat_template_kwargs.__get__(online)
|
||||
)
|
||||
online.render_responses = AsyncMock(
|
||||
return_value=MagicMock(messages=[], engine_input={"prompt_token_ids": [7]})
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert not isinstance(response, ErrorResponse)
|
||||
assert response.reasoning_parser_kwargs is not None
|
||||
kwargs = response.reasoning_parser_kwargs.chat_template_kwargs
|
||||
assert kwargs["enable_thinking"] is False
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = ["--trust-request-chat-template"]
|
||||
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "When should a Responses handler return an empty string?",
|
||||
"max_output_tokens": 7,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["model"] == MODEL_NAME
|
||||
assert data["request_id"].startswith("resp_")
|
||||
assert data["sampling_params"]["max_tokens"] == 7
|
||||
assert data["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_includes_prior_input_items(client):
|
||||
current_turn = {
|
||||
"role": "user",
|
||||
"content": "Which color should I remember?",
|
||||
}
|
||||
current_turn_response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={"model": MODEL_NAME, "input": [current_turn]},
|
||||
)
|
||||
multi_turn_response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": [
|
||||
{"role": "user", "content": "Remember the color cobalt."},
|
||||
{"role": "assistant", "content": "I will remember cobalt."},
|
||||
current_turn,
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert current_turn_response.status_code == 200
|
||||
assert multi_turn_response.status_code == 200
|
||||
current_turn_token_ids = current_turn_response.json()["token_ids"]
|
||||
multi_turn_token_ids = multi_turn_response.json()["token_ids"]
|
||||
assert len(multi_turn_token_ids) > len(current_turn_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_rejects_previous_response_id_over_http(client):
|
||||
response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "Continue the previous response.",
|
||||
"previous_response_id": "resp_previous",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "previous_response_id"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
"""Test basic completion render endpoint."""
|
||||
# Make request to render endpoint
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "When should a chat-completions handler return an empty string?",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure - list of GenerateRequest
|
||||
assert isinstance(data, list)
|
||||
assert len(data) > 0
|
||||
|
||||
# Verify first prompt is a GenerateRequest
|
||||
first_prompt = data[0]
|
||||
assert "token_ids" in first_prompt
|
||||
assert "sampling_params" in first_prompt
|
||||
assert "model" in first_prompt
|
||||
assert "request_id" in first_prompt
|
||||
assert isinstance(first_prompt["token_ids"], list)
|
||||
assert len(first_prompt["token_ids"]) > 0
|
||||
assert first_prompt["model"] == MODEL_NAME
|
||||
assert first_prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_basic(client):
|
||||
"""Test basic chat completion render endpoint."""
|
||||
# Make request to render endpoint
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Returning an empty string for the prompt may be confusing."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure - should be a GenerateRequest
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# Verify token IDs are integers and BOS token is present
|
||||
token_ids = data["token_ids"]
|
||||
assert all(isinstance(tid, int) for tid in token_ids)
|
||||
assert token_ids[0] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts(client):
|
||||
"""Test completion render with multiple prompts."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should return two GenerateRequest items
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
# Verify both prompts have GenerateRequest fields
|
||||
for prompt in data:
|
||||
assert "token_ids" in prompt
|
||||
assert "sampling_params" in prompt
|
||||
assert "model" in prompt
|
||||
assert "request_id" in prompt
|
||||
assert len(prompt["token_ids"]) > 0
|
||||
assert prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
CHAT_BODY = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_honors_body_request_id(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render", json={**CHAT_BODY, "request_id": "abc"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["request_id"] == "chatcmpl-abc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_header_request_id_wins(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={**CHAT_BODY, "request_id": "abc"},
|
||||
headers={"X-Request-Id": "hdr"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["request_id"] == "chatcmpl-hdr"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_generates_unique_request_ids(client):
|
||||
ids = set()
|
||||
for _ in range(2):
|
||||
response = await client.post("/v1/chat/completions/render", json=CHAT_BODY)
|
||||
assert response.status_code == 200
|
||||
request_id = response.json()["request_id"]
|
||||
assert request_id.startswith("chatcmpl-")
|
||||
ids.add(request_id)
|
||||
|
||||
assert len(ids) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_request_id_single_prompt(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={"model": MODEL_NAME, "prompt": "Hello world", "request_id": "abc"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [r["request_id"] for r in response.json()] == ["cmpl-abc"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_request_id_multiple_prompts(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
"request_id": "abc",
|
||||
},
|
||||
headers={"X-Request-Id": "hdr"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [r["request_id"] for r in response.json()] == ["cmpl-hdr-0", "cmpl-hdr-1"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_multi_turn(client):
|
||||
"""Test chat completion render with multi-turn conversation."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify tokenization occurred
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_stream_true(client):
|
||||
"""Render accepts stream params but still returns JSON (non-streamed)."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"stream": True,
|
||||
"stream_options": {
|
||||
"include_usage": True,
|
||||
"continuous_usage_stats": True,
|
||||
},
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stream options should be accepted by /render.",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("content-type", "").startswith("application/json")
|
||||
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# /render should preserve stream fields on the returned token-in request.
|
||||
assert data.get("stream") is True
|
||||
assert isinstance(data.get("stream_options"), dict)
|
||||
assert data["stream_options"].get("include_usage") is True
|
||||
assert data["stream_options"].get("continuous_usage_stats") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_error_invalid_model(client):
|
||||
"""Test completion render with invalid model returns error."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": "invalid-model-name",
|
||||
"prompt": "Hello",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_error_invalid_model(client):
|
||||
"""Test chat completion render with invalid model returns error."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": "invalid-model-name",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_no_generation(client):
|
||||
"""Verify render endpoint does not generate text."""
|
||||
# This test verifies that calling render is fast (no generation)
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Tell me a very long story about " * 10,
|
||||
},
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert response.status_code == 200
|
||||
# Render should be fast (< 1 second) since no generation
|
||||
assert elapsed < 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_sampling_params(client):
|
||||
"""Verify sampling params are correctly returned by /render."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Test sampling params"}],
|
||||
"temperature": 0.123,
|
||||
"top_p": 0.456,
|
||||
"frequency_penalty": 1.1,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert "sampling_params" in data
|
||||
sampling_params = data["sampling_params"]
|
||||
|
||||
assert sampling_params.get("temperature") == 0.123
|
||||
assert sampling_params.get("top_p") == 0.456
|
||||
assert sampling_params.get("frequency_penalty") == 1.1
|
||||
|
||||
# Check that internal fields are not present
|
||||
assert "_all_stop_token_ids" not in sampling_params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_emits_token_offsets(client):
|
||||
"""With return_token_offsets, /v1/completions/render returns per-token
|
||||
(start, end) char offsets aligned with token_ids."""
|
||||
prompt = "Hello, world."
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": prompt,
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, list)
|
||||
offsets = data[0]["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(data[0]["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert isinstance(start, int) and isinstance(end, int)
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_default_no_token_offsets(client):
|
||||
"""Without the flag, token_offsets must be null (existing responses
|
||||
unchanged)."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Hello, world.",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data[0]["token_offsets"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_emits_token_offsets(client):
|
||||
"""With return_token_offsets, /v1/chat/completions/render returns
|
||||
per-token offsets relative to the templated prompt string."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, world."}],
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
offsets = data["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(data["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert isinstance(start, int) and isinstance(end, int)
|
||||
assert 0 <= start <= end
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_default_no_token_offsets(client):
|
||||
"""Without the flag, chat render token_offsets must be null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, world."}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["token_offsets"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_truncated_token_offsets(client):
|
||||
"""Truncation must shorten token_offsets together with token_ids.
|
||||
|
||||
An explicit truncation_side turns off tokenizer-level truncation, so the
|
||||
tokenizer returns offsets for the whole prompt and they are reduced
|
||||
afterwards -- separately from token_ids. GenerateRequest documents that the
|
||||
two lists have equal length.
|
||||
"""
|
||||
prompt = "The quick brown fox jumps over the lazy dog."
|
||||
keep = 4
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": prompt,
|
||||
"return_token_offsets": True,
|
||||
"truncate_prompt_tokens": keep,
|
||||
"truncation_side": "left",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
token_ids = data[0]["token_ids"]
|
||||
offsets = data[0]["token_offsets"]
|
||||
|
||||
assert len(token_ids) == keep
|
||||
assert len(offsets) == len(token_ids)
|
||||
|
||||
# Equal length is not enough: truncating from the left keeps the *last*
|
||||
# tokens, so the surviving offsets must cover the end of the prompt.
|
||||
assert offsets[-1][1] == len(prompt)
|
||||
assert offsets[0][0] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts_token_offsets(client):
|
||||
"""Each prompt in a batch gets its own offsets aligned with its tokens."""
|
||||
prompts = ["Hello, world.", "Goodbye, world."]
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": prompts,
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data) == len(prompts)
|
||||
for item, prompt in zip(data, prompts):
|
||||
offsets = item["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(item["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_render_basic(client):
|
||||
"""Test basic Anthropic Messages render endpoint."""
|
||||
response = await client.post(
|
||||
"/v1/messages/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "Render this Anthropic message."}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Single GenerateRequest, like chat render.
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert "sampling_params" in data
|
||||
assert "model" in data
|
||||
assert data["model"] == MODEL_NAME
|
||||
|
||||
token_ids = data["token_ids"]
|
||||
assert isinstance(token_ids, list)
|
||||
assert len(token_ids) > 0
|
||||
assert all(isinstance(tid, int) for tid in token_ids)
|
||||
assert token_ids[0] == 1 # BOS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_render_system_and_multi_turn(client):
|
||||
"""System field + multi-turn messages render to a single prompt."""
|
||||
response = await client.post(
|
||||
"/v1/messages/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"max_tokens": 16,
|
||||
"system": "You are a helpful assistant.",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi! How can I help?"},
|
||||
{"role": "user", "content": "What is 2 + 2?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
assert len(data["token_ids"]) > 0
|
||||
assert data["token_ids"][0] == 1 # BOS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_render_merges_inline_system(client):
|
||||
"""Inline system messages merge into the leading system block.
|
||||
|
||||
The model's chat template rejects non-leading system messages, so the
|
||||
/v1/messages server path detects merge_inline_system=True and render must
|
||||
produce the same tokens as the manually pre-merged request.
|
||||
"""
|
||||
inline = await client.post(
|
||||
"/v1/messages/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"max_tokens": 16,
|
||||
"system": "You are a helpful assistant.",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi! How can I help?"},
|
||||
{"role": "system", "content": "Be brief."},
|
||||
{"role": "user", "content": "What is 2 + 2?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert inline.status_code == 200
|
||||
|
||||
merged = await client.post(
|
||||
"/v1/messages/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"max_tokens": 16,
|
||||
"system": "You are a helpful assistant.Be brief.",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi! How can I help?"},
|
||||
{"role": "user", "content": "What is 2 + 2?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert merged.status_code == 200
|
||||
|
||||
assert inline.json()["token_ids"] == merged.json()["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_render_error_invalid_model(client):
|
||||
"""Messages render with an invalid model returns an error."""
|
||||
response = await client.post(
|
||||
"/v1/messages/render",
|
||||
json={
|
||||
"model": "invalid-model-name",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Multimodal tests for the /render endpoints that expose prompt preprocessing."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
VISION_MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vision_server():
|
||||
"""Vision-capable server used for multimodal /render tests."""
|
||||
args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"100",
|
||||
"--max-num-seqs",
|
||||
"1",
|
||||
"--limit-mm-per-prompt.image",
|
||||
"1",
|
||||
"--limit-mm-per-prompt.video",
|
||||
"0",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def vision_client(vision_server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=vision_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_base64_image_url(
|
||||
vision_client,
|
||||
local_asset_server,
|
||||
):
|
||||
"""Render a multimodal chat request and verify tokens are returned."""
|
||||
image = local_asset_server.get_image_asset("RGBA_comp.png")
|
||||
data_url = encode_image_url(image, format="PNG")
|
||||
|
||||
assert data_url.startswith("data:image/")
|
||||
assert ";base64," in data_url
|
||||
|
||||
response = await vision_client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# Verify multimodal features are populated
|
||||
assert "features" in data
|
||||
features = data["features"]
|
||||
assert features is not None
|
||||
|
||||
# mm_hashes: should have an "image" key with a list of hash strings
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) for h in image_hashes)
|
||||
|
||||
# mm_placeholders: should have an "image" key with offset/length dicts
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert "offset" in p
|
||||
assert "length" in p
|
||||
assert isinstance(p["offset"], int)
|
||||
assert isinstance(p["length"], int)
|
||||
assert p["length"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_matches_render_for_multimodal_input(
|
||||
vision_client,
|
||||
local_asset_server,
|
||||
):
|
||||
"""`/tokenize` should match `/v1/chat/completions/render` token output."""
|
||||
image = local_asset_server.get_image_asset("RGBA_comp.png")
|
||||
data_url = encode_image_url(image, format="PNG")
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
render_response = await vision_client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": messages,
|
||||
},
|
||||
)
|
||||
assert render_response.status_code == 200
|
||||
render_data = render_response.json()
|
||||
|
||||
tokenize_response = await vision_client.post(
|
||||
"/tokenize",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": messages,
|
||||
},
|
||||
)
|
||||
assert tokenize_response.status_code == 200
|
||||
tokenize_data = tokenize_response.json()
|
||||
|
||||
assert tokenize_data["tokens"] == render_data["token_ids"]
|
||||
assert tokenize_data["count"] == len(render_data["token_ids"])
|
||||
@@ -17,18 +17,10 @@ from vllm.entrypoints.serve.middleware.authenticate import (
|
||||
)
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
"/v1/chat/completions/render",
|
||||
"/v1/chat/completions/derender",
|
||||
"/v1/completions/render",
|
||||
"/v1/completions/derender",
|
||||
"/v1/messages/render",
|
||||
"/v1/responses/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
ABORT_PATH = "/inference/v1/abort_requests"
|
||||
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
|
||||
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
@@ -100,13 +92,6 @@ def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server():
|
||||
paths = registered_paths(("render",))
|
||||
|
||||
assert paths >= RENDER_PATHS
|
||||
assert GENERATE_PATH not in paths
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
@@ -124,8 +109,8 @@ def test_abort_route_without_tokens_only_requires_api_key():
|
||||
|
||||
|
||||
def test_unauthenticated_abort_route_is_not_registered_twice():
|
||||
"""The RLHF dev router registers /abort_requests before the scale-out
|
||||
routers, so attaching the scale-out one too would add a shadowed duplicate."""
|
||||
"""A pre-existing /abort_requests route must not be shadowed by a second
|
||||
registration from the scale-out router."""
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(tokens_only=True)
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
@@ -195,10 +194,6 @@ _MESSAGES = [{"role": "user", "content": "hi"}]
|
||||
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
|
||||
(CompletionRequest, {"model": "m", "prompt": "hi"}),
|
||||
(ResponsesRequest, {"model": "m", "input": "hi"}),
|
||||
(
|
||||
AnthropicMessagesRequest,
|
||||
{"model": "m", "max_tokens": 8, "messages": _MESSAGES},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
|
||||
|
||||
@@ -120,7 +120,6 @@ class TestAffectedModulesUseSanitize:
|
||||
@pytest.mark.parametrize(
|
||||
"module",
|
||||
[
|
||||
"vllm.entrypoints.anthropic.serving",
|
||||
"vllm.entrypoints.speech_to_text.realtime.connection",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -32,7 +32,6 @@ def _build_args() -> Namespace:
|
||||
"""Minimal args for ``build_app``; avoids ``make_arg_parser`` device probing."""
|
||||
return Namespace(
|
||||
disable_fastapi_docs=True,
|
||||
enable_offline_docs=False,
|
||||
root_path=None,
|
||||
allowed_origins=["*"],
|
||||
allow_credentials=False,
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Shared fixtures and utilities for SageMaker tests."""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# Model name constants used across tests
|
||||
MODEL_NAME_SMOLLM = "HuggingFaceTB/SmolLM2-135M-Instruct"
|
||||
LORA_ADAPTER_NAME_SMOLLM = "jekunz/smollm-135m-lora-fineweb-faroese"
|
||||
|
||||
# SageMaker header constants
|
||||
HEADER_SAGEMAKER_CLOSED_SESSION_ID = "X-Amzn-SageMaker-Closed-Session-Id"
|
||||
HEADER_SAGEMAKER_SESSION_ID = "X-Amzn-SageMaker-Session-Id"
|
||||
HEADER_SAGEMAKER_NEW_SESSION_ID = "X-Amzn-SageMaker-New-Session-Id"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def smollm2_lora_files():
|
||||
"""Download LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id=LORA_ADAPTER_NAME_SMOLLM)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def basic_server_with_lora(smollm2_lora_files):
|
||||
"""Basic server fixture with standard configuration."""
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--max-lora-rank",
|
||||
"256",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"64",
|
||||
]
|
||||
|
||||
envs = {
|
||||
"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True",
|
||||
"SAGEMAKER_ENABLE_STATEFUL_SESSIONS": "True",
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME_SMOLLM, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def async_client(basic_server_with_lora: RemoteOpenAIServer):
|
||||
"""Async OpenAI client fixture for use with basic_server."""
|
||||
async with basic_server_with_lora.get_async_client() as async_client:
|
||||
yield async_client
|
||||
@@ -1,629 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Integration tests for handler override functionality.
|
||||
|
||||
Tests real customer usage scenarios:
|
||||
- Using @custom_ping_handler and @custom_invocation_handler decorators
|
||||
to override handlers
|
||||
- Setting environment variables for handler specifications
|
||||
- Writing customer scripts with custom_sagemaker_ping_handler() and
|
||||
custom_sagemaker_invocation_handler() functions
|
||||
- Priority: env vars > decorators > customer script files > framework
|
||||
defaults
|
||||
|
||||
The handler-override scenarios exercise the real vLLM SageMaker router and
|
||||
bootstrap path via an in-process FastAPI ``TestClient`` instead of launching a
|
||||
model server. These scenarios fully replace the ``/ping`` and ``/invocations``
|
||||
endpoints with customer handlers, so no inference engine is required to
|
||||
validate override behavior. Avoiding the model server also keeps the tests
|
||||
fast and deterministic rather than depending on the FastAPI version resolved
|
||||
into the test environment at runtime.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
from .conftest import (
|
||||
MODEL_NAME_SMOLLM,
|
||||
)
|
||||
|
||||
|
||||
def _build_sagemaker_test_client() -> TestClient:
|
||||
"""Build a TestClient over the real SageMaker router and bootstrap path.
|
||||
|
||||
``attach_router`` is called with empty supported tasks because the override
|
||||
tests replace the endpoints with customer handlers, so no framework
|
||||
invocation handler (and therefore no engine) is exercised.
|
||||
"""
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router,
|
||||
sagemaker_standards_bootstrap,
|
||||
)
|
||||
|
||||
app = FastAPI()
|
||||
attach_router(app, ())
|
||||
return TestClient(sagemaker_standards_bootstrap(app))
|
||||
|
||||
|
||||
class TestHandlerOverrideIntegration:
|
||||
"""Integration tests simulating real customer usage scenarios.
|
||||
|
||||
Each test simulates a fresh server startup where customers:
|
||||
- Use @custom_ping_handler and @custom_invocation_handler decorators
|
||||
- Set environment variables (CUSTOM_FASTAPI_PING_HANDLER, etc.)
|
||||
- Write customer scripts with custom_sagemaker_ping_handler() and
|
||||
custom_sagemaker_invocation_handler() functions
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test - simulate fresh server startup."""
|
||||
self._clear_caches()
|
||||
self._clear_env_vars()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test."""
|
||||
self._clear_env_vars()
|
||||
|
||||
def _clear_caches(self):
|
||||
"""Clear handler registry and function loader cache."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.handler import (
|
||||
handler_registry,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.sagemaker_loader import (
|
||||
SageMakerFunctionLoader,
|
||||
)
|
||||
|
||||
handler_registry.clear()
|
||||
SageMakerFunctionLoader._default_function_loader = None
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
def _clear_env_vars(self):
|
||||
"""Clear SageMaker environment variables."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.config import (
|
||||
FastAPIEnvVars,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
|
||||
# Clear SageMaker env vars
|
||||
for var in [
|
||||
SageMakerEnvVars.SAGEMAKER_MODEL_PATH,
|
||||
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME,
|
||||
]:
|
||||
os.environ.pop(var, None)
|
||||
|
||||
# Clear FastAPI env vars
|
||||
for var in [
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
|
||||
]:
|
||||
os.environ.pop(var, None)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
def test_customer_script_functions_auto_loaded(self, monkeypatch, tmp_path):
|
||||
"""Test customer scenario: script functions automatically override
|
||||
framework defaults."""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a script file with ping() and invoke() functions
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
from fastapi import Request
|
||||
|
||||
async def custom_sagemaker_ping_handler():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "customer_override",
|
||||
"message": "Custom ping from customer script"
|
||||
}
|
||||
|
||||
async def custom_sagemaker_invocation_handler(request: Request):
|
||||
return {
|
||||
"predictions": ["Custom response from customer script"],
|
||||
"source": "customer_override"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Customer sets SageMaker environment variables to point to their script
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
# Customer tests their server and sees their overrides work
|
||||
# automatically
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# Customer sees their functions are used
|
||||
assert ping_data["source"] == "customer_override"
|
||||
assert ping_data["message"] == "Custom ping from customer script"
|
||||
assert invoke_data["source"] == "customer_override"
|
||||
assert invoke_data["predictions"] == [
|
||||
"Custom response from customer script"
|
||||
]
|
||||
|
||||
def test_customer_decorator_usage(self, monkeypatch, tmp_path):
|
||||
"""Test customer scenario: using @custom_ping_handler and
|
||||
@custom_invocation_handler decorators."""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a script file with decorators
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
from fastapi import Request
|
||||
|
||||
@sagemaker_standards.custom_ping_handler
|
||||
async def my_ping():
|
||||
return {
|
||||
"type": "ping",
|
||||
"source": "customer_decorator"
|
||||
}
|
||||
|
||||
@sagemaker_standards.custom_invocation_handler
|
||||
async def my_invoke(request: Request):
|
||||
return {
|
||||
"type": "invoke",
|
||||
"source": "customer_decorator"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# Customer sees their handlers are used by the server
|
||||
assert ping_data["source"] == "customer_decorator"
|
||||
assert invoke_data["source"] == "customer_decorator"
|
||||
|
||||
def test_handler_priority_order(self, monkeypatch, tmp_path):
|
||||
"""Test priority: @custom_ping_handler/@custom_invocation_handler
|
||||
decorators vs script functions."""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a script with both decorator and regular functions
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
from fastapi import Request
|
||||
|
||||
# Customer uses @custom_ping_handler decorator (higher priority than script functions)
|
||||
@sagemaker_standards.custom_ping_handler
|
||||
async def decorated_ping():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "ping_decorator_in_script",
|
||||
"priority": "decorator"
|
||||
}
|
||||
|
||||
# Customer also has a regular function (lower priority than
|
||||
# @custom_ping_handler decorator)
|
||||
async def custom_sagemaker_ping_handler():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "script_function",
|
||||
"priority": "function"
|
||||
}
|
||||
|
||||
# Customer has a regular invoke function
|
||||
async def custom_sagemaker_invocation_handler(request: Request):
|
||||
return {
|
||||
"predictions": ["Script function response"],
|
||||
"source": "script_invoke_function",
|
||||
"priority": "function"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# @custom_ping_handler decorator has higher priority than
|
||||
# script function
|
||||
assert ping_data["source"] == "ping_decorator_in_script"
|
||||
assert ping_data["priority"] == "decorator"
|
||||
|
||||
# Script function is used for invoke
|
||||
assert invoke_data["source"] == "script_invoke_function"
|
||||
assert invoke_data["priority"] == "function"
|
||||
|
||||
def test_environment_variable_script_loading(self, monkeypatch, tmp_path):
|
||||
"""Test that environment variables correctly specify script location
|
||||
and loading."""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a script in a specific directory
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
from fastapi import Request
|
||||
|
||||
async def custom_sagemaker_ping_handler():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "env_loaded_script",
|
||||
"method": "environment_variable_loading"
|
||||
}
|
||||
|
||||
async def custom_sagemaker_invocation_handler(request: Request):
|
||||
return {
|
||||
"predictions": ["Loaded via environment variables"],
|
||||
"source": "env_loaded_script",
|
||||
"method": "environment_variable_loading"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Test environment variable script loading
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# Verify that the script was loaded via environment variables
|
||||
assert ping_data["source"] == "env_loaded_script"
|
||||
assert ping_data["method"] == "environment_variable_loading"
|
||||
assert invoke_data["source"] == "env_loaded_script"
|
||||
assert invoke_data["method"] == "environment_variable_loading"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_framework_default_handlers(self):
|
||||
"""Test that framework default handlers work when no customer
|
||||
overrides exist.
|
||||
|
||||
This scenario exercises the real inference path (default
|
||||
``/invocations``), so it keeps using a live model server rather than
|
||||
the in-process TestClient.
|
||||
"""
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
]
|
||||
|
||||
# Explicitly pass empty env_dict to ensure no SageMaker env vars are set
|
||||
# This prevents pollution from previous tests
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.config import (
|
||||
FastAPIEnvVars,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: "",
|
||||
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: "",
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER: "",
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER: "",
|
||||
}
|
||||
except ImportError:
|
||||
env_dict = {}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME_SMOLLM, args, env_dict=env_dict) as server:
|
||||
# Test that default ping works
|
||||
ping_response = requests.get(server.url_for("ping"))
|
||||
assert ping_response.status_code == 200
|
||||
|
||||
# Test that default invocations work
|
||||
invoke_response = requests.post(
|
||||
server.url_for("invocations"),
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
|
||||
def test_handler_env_var_override(self, monkeypatch, tmp_path):
|
||||
"""Test CUSTOM_FASTAPI_PING_HANDLER and CUSTOM_FASTAPI_INVOCATION_HANDLER
|
||||
environment variable overrides."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.config import (
|
||||
FastAPIEnvVars,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Create a script with both env var handlers and script functions
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
from fastapi import Request, Response
|
||||
import json
|
||||
|
||||
async def env_var_ping_handler(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"status": "healthy",
|
||||
"source": "env_var_ping",
|
||||
"method": "environment_variable"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
async def env_var_invoke_handler(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"predictions": ["Environment variable response"],
|
||||
"source": "env_var_invoke",
|
||||
"method": "environment_variable"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
async def custom_sagemaker_ping_handler():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "script_ping",
|
||||
"method": "script_function"
|
||||
}
|
||||
|
||||
async def custom_sagemaker_invocation_handler(request: Request):
|
||||
return {
|
||||
"predictions": ["Script function response"],
|
||||
"source": "script_invoke",
|
||||
"method": "script_function"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Set environment variables to override both handlers
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
monkeypatch.setenv(
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
|
||||
f"{script_path.name}:env_var_ping_handler",
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
|
||||
f"{script_path.name}:env_var_invoke_handler",
|
||||
)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
# Test ping handler override
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
# Environment variable should override script function
|
||||
assert ping_data["method"] == "environment_variable"
|
||||
assert ping_data["source"] == "env_var_ping"
|
||||
|
||||
# Test invocation handler override
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# Environment variable should override script function
|
||||
assert invoke_data["method"] == "environment_variable"
|
||||
assert invoke_data["source"] == "env_var_invoke"
|
||||
|
||||
def test_env_var_priority_over_decorator_and_script(self, monkeypatch, tmp_path):
|
||||
"""Test that environment variables have highest priority over decorators
|
||||
and script functions for both ping and invocation handlers."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.config import (
|
||||
FastAPIEnvVars,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Create a script with all three handler types for both ping and invocation
|
||||
script_path = tmp_path / "model.py"
|
||||
script_path.write_text(
|
||||
"""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
from fastapi import Request, Response
|
||||
import json
|
||||
|
||||
# Environment variable handlers (highest priority)
|
||||
async def env_priority_ping(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"status": "healthy",
|
||||
"source": "env_var",
|
||||
"priority": "environment_variable"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
async def env_priority_invoke(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"predictions": ["Environment variable response"],
|
||||
"source": "env_var",
|
||||
"priority": "environment_variable"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
# Decorator handlers (medium priority)
|
||||
@sagemaker_standards.custom_ping_handler
|
||||
async def decorator_ping(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"status": "healthy",
|
||||
"source": "decorator",
|
||||
"priority": "decorator"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
@sagemaker_standards.custom_invocation_handler
|
||||
async def decorator_invoke(raw_request: Request) -> Response:
|
||||
return Response(
|
||||
content=json.dumps({
|
||||
"predictions": ["Decorator response"],
|
||||
"source": "decorator",
|
||||
"priority": "decorator"
|
||||
}),
|
||||
media_type="application/json"
|
||||
)
|
||||
|
||||
# Script functions (lowest priority)
|
||||
async def custom_sagemaker_ping_handler():
|
||||
return {
|
||||
"status": "healthy",
|
||||
"source": "script",
|
||||
"priority": "script_function"
|
||||
}
|
||||
|
||||
async def custom_sagemaker_invocation_handler(request: Request):
|
||||
return {
|
||||
"predictions": ["Script function response"],
|
||||
"source": "script",
|
||||
"priority": "script_function"
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
# Set environment variables to specify highest priority handlers
|
||||
monkeypatch.setenv(SageMakerEnvVars.SAGEMAKER_MODEL_PATH, str(tmp_path))
|
||||
monkeypatch.setenv(SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME, script_path.name)
|
||||
monkeypatch.setenv(
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_PING_HANDLER,
|
||||
f"{script_path.name}:env_priority_ping",
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_INVOCATION_HANDLER,
|
||||
f"{script_path.name}:env_priority_invoke",
|
||||
)
|
||||
|
||||
with _build_sagemaker_test_client() as client:
|
||||
# Test ping handler priority
|
||||
ping_response = client.get("/ping")
|
||||
assert ping_response.status_code == 200
|
||||
ping_data = ping_response.json()
|
||||
|
||||
# Environment variable has highest priority and should be used
|
||||
assert ping_data["priority"] == "environment_variable"
|
||||
assert ping_data["source"] == "env_var"
|
||||
|
||||
# Test invocation handler priority
|
||||
invoke_response = client.post(
|
||||
"/invocations",
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
},
|
||||
)
|
||||
assert invoke_response.status_code == 200
|
||||
invoke_data = invoke_response.json()
|
||||
|
||||
# Environment variable has highest priority and should be used
|
||||
assert invoke_data["priority"] == "environment_variable"
|
||||
assert invoke_data["source"] == "env_var"
|
||||
@@ -1,172 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import openai # use the official async_client for correctness check
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
from .conftest import MODEL_NAME_SMOLLM
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_load_adapter_happy_path(
|
||||
async_client: openai.AsyncOpenAI,
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
smollm2_lora_files,
|
||||
):
|
||||
# The SageMaker standards library creates a POST /adapters endpoint
|
||||
# that maps to the load_lora_adapter handler with request shape:
|
||||
# {"lora_name": "body.name", "lora_path": "body.src"}
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": "smollm2-lora-sagemaker", "src": smollm2_lora_files},
|
||||
)
|
||||
load_response.raise_for_status()
|
||||
|
||||
models = await async_client.models.list()
|
||||
models = models.data
|
||||
dynamic_lora_model = models[-1]
|
||||
assert dynamic_lora_model.root == smollm2_lora_files
|
||||
assert dynamic_lora_model.parent == MODEL_NAME_SMOLLM
|
||||
assert dynamic_lora_model.id == "smollm2-lora-sagemaker"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_unload_adapter_happy_path(
|
||||
async_client: openai.AsyncOpenAI,
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
smollm2_lora_files,
|
||||
):
|
||||
# First, load an adapter
|
||||
adapter_name = "smollm2-lora-sagemaker-unload"
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": adapter_name, "src": smollm2_lora_files},
|
||||
)
|
||||
load_response.raise_for_status()
|
||||
|
||||
# Verify it's in the models list
|
||||
models = await async_client.models.list()
|
||||
adapter_ids = [model.id for model in models.data]
|
||||
assert adapter_name in adapter_ids
|
||||
|
||||
# Now unload it using DELETE /adapters/{adapter_name}
|
||||
# The SageMaker standards maps this to unload_lora_adapter with:
|
||||
# {"lora_name": "path_params.adapter_name"}
|
||||
unload_response = requests.delete(
|
||||
basic_server_with_lora.url_for("adapters", adapter_name),
|
||||
)
|
||||
unload_response.raise_for_status()
|
||||
|
||||
# Verify it's no longer in the models list
|
||||
models = await async_client.models.list()
|
||||
adapter_ids = [model.id for model in models.data]
|
||||
assert adapter_name not in adapter_ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_load_adapter_not_found(
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
):
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": "nonexistent-adapter", "src": "/path/does/not/exist"},
|
||||
)
|
||||
assert load_response.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_load_adapter_invalid_files(
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
tmp_path,
|
||||
):
|
||||
invalid_files = tmp_path / "invalid_adapter"
|
||||
invalid_files.mkdir()
|
||||
(invalid_files / "adapter_config.json").write_text("not valid json")
|
||||
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": "invalid-adapter", "src": str(invalid_files)},
|
||||
)
|
||||
assert load_response.status_code == 500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_unload_nonexistent_adapter(
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
):
|
||||
# Attempt to unload an adapter that doesn't exist
|
||||
unload_response = requests.delete(
|
||||
basic_server_with_lora.url_for("adapters", "nonexistent-adapter-name"),
|
||||
)
|
||||
assert unload_response.status_code in (400, 404)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_invocations_with_adapter(
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
smollm2_lora_files,
|
||||
):
|
||||
# First, load an adapter via SageMaker endpoint
|
||||
adapter_name = "smollm2-lora-invoke-test"
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": adapter_name, "src": smollm2_lora_files},
|
||||
)
|
||||
load_response.raise_for_status()
|
||||
|
||||
# Now test the /invocations endpoint with the adapter
|
||||
invocation_response = requests.post(
|
||||
basic_server_with_lora.url_for("invocations"),
|
||||
headers={
|
||||
"X-Amzn-SageMaker-Adapter-Identifier": adapter_name,
|
||||
},
|
||||
json={
|
||||
"prompt": "Hello, how are you?",
|
||||
"max_tokens": 10,
|
||||
},
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
# Verify we got a valid completion response
|
||||
assert "choices" in invocation_output
|
||||
assert len(invocation_output["choices"]) > 0
|
||||
assert "text" in invocation_output["choices"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_multiple_adapters_load_unload(
|
||||
async_client: openai.AsyncOpenAI,
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
smollm2_lora_files,
|
||||
):
|
||||
adapter_names = [f"sagemaker-adapter-{i}" for i in range(5)]
|
||||
|
||||
# Load all adapters
|
||||
for adapter_name in adapter_names:
|
||||
load_response = requests.post(
|
||||
basic_server_with_lora.url_for("adapters"),
|
||||
json={"name": adapter_name, "src": smollm2_lora_files},
|
||||
)
|
||||
load_response.raise_for_status()
|
||||
|
||||
# Verify all are in the models list
|
||||
models = await async_client.models.list()
|
||||
adapter_ids = [model.id for model in models.data]
|
||||
for adapter_name in adapter_names:
|
||||
assert adapter_name in adapter_ids
|
||||
|
||||
# Unload all adapters
|
||||
for adapter_name in adapter_names:
|
||||
unload_response = requests.delete(
|
||||
basic_server_with_lora.url_for("adapters", adapter_name),
|
||||
)
|
||||
unload_response.raise_for_status()
|
||||
|
||||
# Verify all are removed from models list
|
||||
models = await async_client.models.list()
|
||||
adapter_ids = [model.id for model in models.data]
|
||||
for adapter_name in adapter_names:
|
||||
assert adapter_name not in adapter_ids
|
||||
@@ -1,347 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Integration test for middleware loader functionality.
|
||||
|
||||
Tests that customer middlewares get called correctly with a vLLM server.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
from .conftest import (
|
||||
MODEL_NAME_SMOLLM,
|
||||
)
|
||||
|
||||
|
||||
class TestMiddlewareIntegration:
|
||||
"""Integration test for middleware with vLLM server."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test - simulate fresh server startup."""
|
||||
self._clear_caches()
|
||||
|
||||
def _clear_caches(self):
|
||||
"""Clear middleware registry and function loader cache."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.middleware import (
|
||||
middleware_registry,
|
||||
)
|
||||
from model_hosting_container_standards.common.fastapi.middleware.source.decorator_loader import ( # noqa: E501
|
||||
decorator_loader,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.sagemaker_loader import (
|
||||
SageMakerFunctionLoader,
|
||||
)
|
||||
|
||||
middleware_registry.clear_middlewares()
|
||||
decorator_loader.clear()
|
||||
SageMakerFunctionLoader._default_function_loader = None
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_customer_middleware_with_vllm_server(self):
|
||||
"""Test that customer middlewares work with actual vLLM server.
|
||||
|
||||
Tests decorator-based middlewares (@custom_middleware, @input_formatter,
|
||||
@output_formatter)
|
||||
on multiple endpoints (chat/completions, invocations).
|
||||
"""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a middleware script with multiple decorators
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
||||
f.write(
|
||||
"""
|
||||
from model_hosting_container_standards.common.fastapi.middleware import (
|
||||
custom_middleware, input_formatter, output_formatter
|
||||
)
|
||||
|
||||
# Global flag to track if input formatter was called
|
||||
_input_formatter_called = False
|
||||
|
||||
@input_formatter
|
||||
async def customer_input_formatter(request):
|
||||
# Process input - mark that input formatter was called
|
||||
global _input_formatter_called
|
||||
_input_formatter_called = True
|
||||
return request
|
||||
|
||||
@custom_middleware("throttle")
|
||||
async def customer_throttle_middleware(request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Customer-Throttle"] = "applied"
|
||||
order = response.headers.get("X-Middleware-Order", "")
|
||||
response.headers["X-Middleware-Order"] = order + "throttle,"
|
||||
return response
|
||||
|
||||
@output_formatter
|
||||
async def customer_output_formatter(response):
|
||||
global _input_formatter_called
|
||||
response.headers["X-Customer-Processed"] = "true"
|
||||
# Since input_formatter and output_formatter are combined into
|
||||
# pre_post_process middleware,
|
||||
# if output_formatter is called, input_formatter should have been called too
|
||||
if _input_formatter_called:
|
||||
response.headers["X-Input-Formatter-Called"] = "true"
|
||||
order = response.headers.get("X-Middleware-Order", "")
|
||||
response.headers["X-Middleware-Order"] = order + "output_formatter,"
|
||||
return response
|
||||
"""
|
||||
)
|
||||
script_path = f.name
|
||||
|
||||
try:
|
||||
script_dir = os.path.dirname(script_path)
|
||||
script_name = os.path.basename(script_path)
|
||||
|
||||
# Set environment variables to point to customer script
|
||||
env_vars = {
|
||||
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
|
||||
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
|
||||
}
|
||||
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME_SMOLLM, args, env_dict=env_vars
|
||||
) as server:
|
||||
# Test 1: Middlewares applied to chat/completions endpoint
|
||||
chat_response = requests.post(
|
||||
server.url_for("v1/chat/completions"),
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
},
|
||||
)
|
||||
|
||||
assert chat_response.status_code == 200
|
||||
|
||||
# Verify all middlewares were executed
|
||||
assert "X-Customer-Throttle" in chat_response.headers
|
||||
assert chat_response.headers["X-Customer-Throttle"] == "applied"
|
||||
assert "X-Customer-Processed" in chat_response.headers
|
||||
assert chat_response.headers["X-Customer-Processed"] == "true"
|
||||
|
||||
# Verify input formatter was called
|
||||
assert "X-Input-Formatter-Called" in chat_response.headers
|
||||
assert chat_response.headers["X-Input-Formatter-Called"] == "true"
|
||||
|
||||
# Verify middleware execution order
|
||||
execution_order = chat_response.headers.get(
|
||||
"X-Middleware-Order", ""
|
||||
).rstrip(",")
|
||||
order_parts = execution_order.split(",") if execution_order else []
|
||||
assert "throttle" in order_parts
|
||||
assert "output_formatter" in order_parts
|
||||
|
||||
# Test 2: Middlewares applied to invocations endpoint
|
||||
invocations_response = requests.post(
|
||||
server.url_for("invocations"),
|
||||
json={
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
},
|
||||
)
|
||||
|
||||
assert invocations_response.status_code == 200
|
||||
|
||||
# Verify all middlewares were executed
|
||||
assert "X-Customer-Throttle" in invocations_response.headers
|
||||
assert invocations_response.headers["X-Customer-Throttle"] == "applied"
|
||||
assert "X-Customer-Processed" in invocations_response.headers
|
||||
assert invocations_response.headers["X-Customer-Processed"] == "true"
|
||||
|
||||
# Verify input formatter was called
|
||||
assert "X-Input-Formatter-Called" in invocations_response.headers
|
||||
assert (
|
||||
invocations_response.headers["X-Input-Formatter-Called"] == "true"
|
||||
)
|
||||
|
||||
finally:
|
||||
os.unlink(script_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_middleware_with_ping_endpoint(self):
|
||||
"""Test that middlewares work with SageMaker ping endpoint."""
|
||||
try:
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Customer writes a middleware script
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
||||
f.write(
|
||||
"""
|
||||
from model_hosting_container_standards.common.fastapi.middleware import (
|
||||
custom_middleware
|
||||
)
|
||||
|
||||
@custom_middleware("pre_post_process")
|
||||
async def ping_tracking_middleware(request, call_next):
|
||||
response = await call_next(request)
|
||||
if request.url.path == "/ping":
|
||||
response.headers["X-Ping-Tracked"] = "true"
|
||||
return response
|
||||
"""
|
||||
)
|
||||
script_path = f.name
|
||||
|
||||
try:
|
||||
script_dir = os.path.dirname(script_path)
|
||||
script_name = os.path.basename(script_path)
|
||||
|
||||
env_vars = {
|
||||
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
|
||||
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
|
||||
}
|
||||
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME_SMOLLM, args, env_dict=env_vars
|
||||
) as server:
|
||||
# Test ping endpoint with middleware
|
||||
response = requests.get(server.url_for("ping"))
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "X-Ping-Tracked" in response.headers
|
||||
assert response.headers["X-Ping-Tracked"] == "true"
|
||||
|
||||
finally:
|
||||
os.unlink(script_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_middleware_env_var_override(self):
|
||||
"""Test middleware environment variable overrides."""
|
||||
try:
|
||||
from model_hosting_container_standards.common.fastapi.config import (
|
||||
FastAPIEnvVars,
|
||||
)
|
||||
from model_hosting_container_standards.sagemaker.config import (
|
||||
SageMakerEnvVars,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("model-hosting-container-standards not available")
|
||||
|
||||
# Create a script with middleware functions specified via env vars
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
||||
f.write(
|
||||
"""
|
||||
from fastapi import Request
|
||||
|
||||
# Global flag to track if pre_process was called
|
||||
_pre_process_called = False
|
||||
|
||||
async def env_throttle_middleware(request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Env-Throttle"] = "applied"
|
||||
return response
|
||||
|
||||
async def env_pre_process(request: Request) -> Request:
|
||||
# Mark that pre_process was called
|
||||
global _pre_process_called
|
||||
_pre_process_called = True
|
||||
return request
|
||||
|
||||
async def env_post_process(response):
|
||||
global _pre_process_called
|
||||
if hasattr(response, 'headers'):
|
||||
response.headers["X-Env-Post-Process"] = "applied"
|
||||
# Since pre_process and post_process are combined into
|
||||
# pre_post_process middleware,
|
||||
# if post_process is called, pre_process should have been called too
|
||||
if _pre_process_called:
|
||||
response.headers["X-Pre-Process-Called"] = "true"
|
||||
return response
|
||||
"""
|
||||
)
|
||||
script_path = f.name
|
||||
|
||||
try:
|
||||
script_dir = os.path.dirname(script_path)
|
||||
script_name = os.path.basename(script_path)
|
||||
|
||||
# Set environment variables for middleware
|
||||
# Use script_name with .py extension as per plugin example
|
||||
env_vars = {
|
||||
SageMakerEnvVars.SAGEMAKER_MODEL_PATH: script_dir,
|
||||
SageMakerEnvVars.CUSTOM_SCRIPT_FILENAME: script_name,
|
||||
FastAPIEnvVars.CUSTOM_FASTAPI_MIDDLEWARE_THROTTLE: (
|
||||
f"{script_name}:env_throttle_middleware"
|
||||
),
|
||||
FastAPIEnvVars.CUSTOM_PRE_PROCESS: f"{script_name}:env_pre_process",
|
||||
FastAPIEnvVars.CUSTOM_POST_PROCESS: f"{script_name}:env_post_process",
|
||||
}
|
||||
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME_SMOLLM, args, env_dict=env_vars
|
||||
) as server:
|
||||
response = requests.get(server.url_for("ping"))
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check if environment variable middleware was applied
|
||||
headers = response.headers
|
||||
|
||||
# Verify that env var middlewares were applied
|
||||
assert "X-Env-Throttle" in headers, (
|
||||
"Throttle middleware should be applied via env var"
|
||||
)
|
||||
assert headers["X-Env-Throttle"] == "applied"
|
||||
|
||||
assert "X-Env-Post-Process" in headers, (
|
||||
"Post-process middleware should be applied via env var"
|
||||
)
|
||||
assert headers["X-Env-Post-Process"] == "applied"
|
||||
|
||||
# Verify that pre_process was called
|
||||
assert "X-Pre-Process-Called" in headers, (
|
||||
"Pre-process should be called via env var"
|
||||
)
|
||||
assert headers["X-Pre-Process-Called"] == "true"
|
||||
|
||||
finally:
|
||||
os.unlink(script_path)
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
from .conftest import (
|
||||
HEADER_SAGEMAKER_CLOSED_SESSION_ID,
|
||||
HEADER_SAGEMAKER_NEW_SESSION_ID,
|
||||
HEADER_SAGEMAKER_SESSION_ID,
|
||||
MODEL_NAME_SMOLLM,
|
||||
)
|
||||
|
||||
CLOSE_BADREQUEST_CASES = [
|
||||
(
|
||||
"nonexistent_session_id",
|
||||
{"session_id": "nonexistent-session-id"},
|
||||
{},
|
||||
"session not found",
|
||||
),
|
||||
("malformed_close_request", {}, {"extra-field": "extra-field-data"}, None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_session_badrequest(basic_server_with_lora: RemoteOpenAIServer):
|
||||
bad_response = requests.post(
|
||||
basic_server_with_lora.url_for("invocations"),
|
||||
json={"requestType": "NEW_SESSION", "extra-field": "extra-field-data"},
|
||||
)
|
||||
|
||||
assert bad_response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"test_name,session_id_change,request_body_change,expected_error",
|
||||
CLOSE_BADREQUEST_CASES,
|
||||
)
|
||||
async def test_close_session_badrequest(
|
||||
basic_server_with_lora: RemoteOpenAIServer,
|
||||
test_name: str,
|
||||
session_id_change: dict[str, str],
|
||||
request_body_change: dict[str, str],
|
||||
expected_error: str | None,
|
||||
):
|
||||
# first attempt to create a session
|
||||
url = basic_server_with_lora.url_for("invocations")
|
||||
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
|
||||
create_response.raise_for_status()
|
||||
valid_session_id, expiration = create_response.headers.get(
|
||||
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
|
||||
).split(";")
|
||||
assert valid_session_id
|
||||
|
||||
close_request_json = {"requestType": "CLOSE"}
|
||||
if request_body_change:
|
||||
close_request_json.update(request_body_change)
|
||||
bad_session_id = session_id_change.get("session_id")
|
||||
bad_close_response = requests.post(
|
||||
url,
|
||||
headers={HEADER_SAGEMAKER_SESSION_ID: bad_session_id or valid_session_id},
|
||||
json=close_request_json,
|
||||
)
|
||||
|
||||
# clean up created session, should succeed
|
||||
clean_up_response = requests.post(
|
||||
url,
|
||||
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
|
||||
json={"requestType": "CLOSE"},
|
||||
)
|
||||
clean_up_response.raise_for_status()
|
||||
|
||||
assert bad_close_response.status_code == 400
|
||||
if expected_error:
|
||||
assert expected_error in bad_close_response.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_session_invalidrequest(
|
||||
basic_server_with_lora: RemoteOpenAIServer, async_client: openai.AsyncOpenAI
|
||||
):
|
||||
# first attempt to create a session
|
||||
url = basic_server_with_lora.url_for("invocations")
|
||||
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
|
||||
create_response.raise_for_status()
|
||||
valid_session_id, expiration = create_response.headers.get(
|
||||
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
|
||||
).split(";")
|
||||
assert valid_session_id
|
||||
|
||||
close_request_json = {"requestType": "CLOSE"}
|
||||
invalid_close_response = requests.post(
|
||||
url,
|
||||
# no headers to specify session_id
|
||||
json=close_request_json,
|
||||
)
|
||||
|
||||
# clean up created session, should succeed
|
||||
clean_up_response = requests.post(
|
||||
url,
|
||||
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
|
||||
json={"requestType": "CLOSE"},
|
||||
)
|
||||
clean_up_response.raise_for_status()
|
||||
|
||||
assert invalid_close_response.status_code == 424
|
||||
assert "invalid session_id" in invalid_close_response.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session(basic_server_with_lora: RemoteOpenAIServer):
|
||||
# first attempt to create a session
|
||||
url = basic_server_with_lora.url_for("invocations")
|
||||
create_response = requests.post(url, json={"requestType": "NEW_SESSION"})
|
||||
create_response.raise_for_status()
|
||||
valid_session_id, expiration = create_response.headers.get(
|
||||
HEADER_SAGEMAKER_NEW_SESSION_ID, ""
|
||||
).split(";")
|
||||
assert valid_session_id
|
||||
|
||||
# test invocation with session id
|
||||
|
||||
request_args = {
|
||||
"model": MODEL_NAME_SMOLLM,
|
||||
"prompt": "what is 1+1?",
|
||||
"max_completion_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
"logprobs": False,
|
||||
}
|
||||
|
||||
invocation_response = requests.post(
|
||||
basic_server_with_lora.url_for("invocations"),
|
||||
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
|
||||
json=request_args,
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
# close created session, should succeed
|
||||
close_response = requests.post(
|
||||
url,
|
||||
headers={HEADER_SAGEMAKER_SESSION_ID: valid_session_id},
|
||||
json={"requestType": "CLOSE"},
|
||||
)
|
||||
close_response.raise_for_status()
|
||||
|
||||
assert (
|
||||
close_response.headers.get(HEADER_SAGEMAKER_CLOSED_SESSION_ID)
|
||||
== valid_session_id
|
||||
)
|
||||
@@ -6,7 +6,7 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
import tests.utils as test_utils
|
||||
from tests.utils import RemoteLaunchRenderServer, RemoteOpenAIServer
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -30,7 +30,6 @@ def test_redact_sensitive_cli_arg_variants(args: list[str], expected: list[str])
|
||||
("server_cls", "command"),
|
||||
[
|
||||
(RemoteOpenAIServer, ["vllm", "serve"]),
|
||||
(RemoteLaunchRenderServer, ["vllm", "launch", "render"]),
|
||||
],
|
||||
)
|
||||
def test_server_redacts_sensitive_values(
|
||||
|
||||
Reference in New Issue
Block a user