[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
-34
View File
@@ -65,37 +65,3 @@ def sample_structured_outputs_choices():
"Swift",
"Kotlin",
]
@pytest.fixture(scope="session")
def qwen3_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
@pytest.fixture(scope="session")
def qwen3_woofing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def opt125_lora_files() -> str:
"""Download opt-125m LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(
repo_id="peft-internal-testing/opt-125m-dummy-lora"
)
+4 -124
View File
@@ -1,28 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.utils import VLLM_PATH
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.exceptions import VLLMValidationError
from vllm.utils.argparse_utils import FlexibleArgumentParser
LORA_MODULE = {
"name": "module2",
"path": "/path/to/module2",
"base_model_name": "llama",
}
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
assert CHATML_JINJA_PATH.exists()
def _build_vllm_parsers():
vllm_parser = FlexibleArgumentParser()
@@ -96,87 +84,6 @@ def test_logging_config_cli_args(serve_parser):
assert config.pylogging_config_file == "/tmp/flat.json"
### Tests for LoRA module parsing
def test_valid_key_value_format(serve_parser):
# Test old format: name=path
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
]
)
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
assert args.lora_modules == expected
def test_valid_json_format(serve_parser):
# Test valid JSON format input
args = serve_parser.parse_args(
[
"--lora-modules",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
]
assert args.lora_modules == expected
def test_invalid_json_format(serve_parser):
# Test invalid JSON format input, missing closing brace
with pytest.raises(SystemExit):
serve_parser.parse_args(
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
)
def test_invalid_type_error(serve_parser):
# Test type error when values are not JSON or key=value
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
"invalid_format", # This is not JSON or key=value format
]
)
def test_invalid_json_field(serve_parser):
# Test valid JSON format but missing required fields
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
'{"name": "module4"}', # Missing required 'path' field
]
)
def test_empty_values(serve_parser):
# Test when no LoRA modules are provided
args = serve_parser.parse_args(["--lora-modules", ""])
assert args.lora_modules == []
def test_multiple_valid_inputs(serve_parser):
# Test multiple valid inputs (both old and JSON format)
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module1", path="/path/to/module1"),
LoRAModulePath(
name="module2", path="/path/to/module2", base_model_name="llama"
),
]
assert args.lora_modules == expected
### Tests for serve argument validation that run prior to loading
@@ -241,10 +148,12 @@ def test_reasoning_parser_none_disables_parser(serve_parser, value, expected):
assert args.reasoning_parser == expected
def test_chat_template_validation_for_happy_paths(serve_parser):
def test_chat_template_validation_for_happy_paths(serve_parser, tmp_path):
"""Ensure validation passes if the chat template exists."""
chat_template = tmp_path / "template.jinja"
chat_template.write_text("{{ messages }}")
args = serve_parser.parse_args(
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
args=["--chat-template", chat_template.absolute().as_posix()]
)
validate_parsed_serve_args(args)
@@ -389,32 +298,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
else:
with pytest.raises(raises):
vllm_parser.parse_args(args=args)
### Tests for LoRA target modules parsing
def test_lora_target_modules_single(serve_parser):
"""Test parsing single lora-target-modules argument."""
args = serve_parser.parse_args(
args=["--enable-lora", "--lora-target-modules", "o_proj"]
)
assert args.lora_target_modules == ["o_proj"]
def test_lora_target_modules_multiple(serve_parser):
"""Test parsing multiple lora-target-modules arguments."""
args = serve_parser.parse_args(
args=[
"--enable-lora",
"--lora-target-modules",
"o_proj",
"qkv_proj",
"down_proj",
]
)
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
def test_lora_target_modules_default_none(serve_parser):
"""Test that lora-target-modules defaults to None."""
args = serve_parser.parse_args(args=[])
assert args.lora_target_modules is None
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.conftest import AudioTestAssets
from tests.utils import RemoteOpenAIServer
from vllm.transformers_utils.repo_utils import hf_api
# NOTE - the tests in this module are currently analogous to test_chat, but are
# separated to avoid OOM killing due to module-scoped servers, since we
# need a multimodal model for these tests.
# Contains a modality specific lora alongside the base model
MULTIMODAL_MODEL_NAME = hf_api().snapshot_download(
"microsoft/Phi-4-multimodal-instruct"
)
AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
@pytest.fixture(scope="module")
def multimodal_server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"half",
"--max-model-len",
"4096",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"speech={AUDIO_LORA_PATH}",
"--max-lora-rank",
"320",
"--max-num-seqs",
"2",
"--trust-remote-code",
"--gpu-memory-utilization",
"0.8",
"--default-mm-loras",
f'{{"audio": "{AUDIO_LORA_PATH}"}}',
]
with RemoteOpenAIServer(
MULTIMODAL_MODEL_NAME, args, max_wait_seconds=480
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def multi_modal_client(multimodal_server):
async with multimodal_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
# base model with default lora should give the same response as lora model
"model_name",
[MULTIMODAL_MODEL_NAME, "speech"],
)
async def test_default_mm_lora_chat_completions(
model_name: str,
multi_modal_client: openai.AsyncOpenAI,
audio_assets: AudioTestAssets,
):
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Can you transcribe this audio?",
},
{
"type": "audio_url",
"audio_url": {"url": audio_assets[0].url},
},
],
}
]
chat_completion = await multi_modal_client.chat.completions.create(
model=model_name, messages=messages, max_completion_tokens=128, temperature=0.0
)
assert len(chat_completion.choices) > 0
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
assert message.content == ACTIVE_MM_LORA_RESPONSE
@@ -27,15 +27,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@pytest.fixture(scope="module")
def zephyr_lora_files():
"""Download zephyr LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
@pytest.fixture(scope="module")
def server(zephyr_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -43,14 +35,6 @@ def server(zephyr_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"zephyr-lora={zephyr_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -67,9 +51,8 @@ async def client(server):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# first test base model, then test loras
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -91,9 +74,8 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -119,7 +101,7 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -145,7 +127,7 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -254,7 +236,7 @@ async def test_more_than_one_prompt_logprobs_chat(
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -297,9 +279,8 @@ async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -345,7 +326,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
["HuggingFaceH4/zephyr-7b-beta"],
)
async def test_chat_completion_stream_options(
client: openai.AsyncOpenAI, model_name: str
@@ -9,7 +9,6 @@ import openai # use the official client for correctness check
import pytest
import pytest_asyncio
# downloading lora to test lora requests
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
# any model with a chat template should work here
+2 -15
View File
@@ -9,12 +9,10 @@ from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
# generation quality here
@pytest.fixture(scope="module")
def server(qwen3_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -22,14 +20,6 @@ def server(qwen3_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"qwen3-lora={qwen3_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -45,12 +35,9 @@ async def client(server):
@pytest.mark.asyncio
async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
async def test_check_models(client: openai.AsyncOpenAI):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token-offsets request/response protocol wiring:
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
serialization boundary. Plain Pydantic field storage is not retested here.
"""
from unittest.mock import Mock
from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams
def _model_config() -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = 128
return model_config
def test_completion_flag_forwarded_to_tok_params():
"""build_tok_params must forward return_token_offsets, defaulting to
False (zero behavioral change for existing callers) and coercing JSON
null to False via the bool() guard."""
cfg = _model_config()
default = CompletionRequest(model="m", prompt="hi")
assert default.build_tok_params(cfg).return_token_offsets is False
on = CompletionRequest(model="m", prompt="hi", return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = CompletionRequest(model="m", prompt="hi", return_token_offsets=None)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_chat_flag_forwarded_to_tok_params():
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
its return_token_offsets forwarding is verified independently."""
cfg = _model_config()
messages = [{"role": "user", "content": "hi"}]
default = ChatCompletionRequest(model="m", messages=messages)
assert default.build_tok_params(cfg).return_token_offsets is False
on = ChatCompletionRequest(model="m", messages=messages, return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = ChatCompletionRequest(
model="m", messages=messages, return_token_offsets=None
)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_generate_request_token_offsets_default_none():
"""Defaults to None so existing /v1/.../render responses are unchanged."""
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=SamplingParams())
assert req.token_offsets is None
def test_generate_request_token_offsets_survive_json_round_trip():
"""GenerateRequest crosses the disagg serialization boundary; the
tuple[int, int] offsets must survive model_dump and re-validate."""
req = GenerateRequest(
token_ids=[10, 20],
sampling_params=SamplingParams(),
token_offsets=[(0, 1), (1, 3)],
)
dumped = req.model_dump()
assert dumped["token_offsets"] == [(0, 1), (1, 3)]
# Re-validate from the dumped dict (sampling_params doesn't round-trip
# cleanly via dump, so re-inject a fresh instance).
again = GenerateRequest.model_validate(
{**dumped, "sampling_params": SamplingParams()}
)
assert again.token_offsets == [(0, 1), (1, 3)]
@@ -1,125 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from argparse import Namespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import (
GUARDED_PREFIX,
AuthenticationMiddleware,
)
from vllm.tasks import SupportedTask
GENERATE_PATH = "/inference/v1/generate"
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...],
*,
tokens_only: bool = False,
enable_scale_out: bool = False,
) -> set[str]:
app = FastAPI()
args = Namespace(
tokens_only=tokens_only,
enable_scale_out=enable_scale_out,
enable_fault_tolerance=False,
)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
@pytest.mark.parametrize("authorized", [False, True])
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
app = FastAPI()
register_scale_out_api_routers(app, ("generate",))
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
headers = {"Authorization": "Bearer secret"} if authorized else {}
client = TestClient(app)
response = client.post(path, content="not JSON", headers=headers)
assert response.status_code == (403 if authorized else 401)
if authorized:
assert "--enable-scale-out" in response.json()["error"]["message"]
assert response.json()["error"]["type"] == "PermissionDeniedError"
assert path not in app.openapi()["paths"]
assert client.post("/unknown", headers=headers).status_code == 404
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert [
record.levelno
for record in caplog.records
if "--enable-scale-out" in record.message
] == [logging.INFO]
@pytest.mark.parametrize("tokens_only", [False, True])
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
app = FastAPI()
app.state.args = Namespace(
tokens_only=tokens_only, enable_scale_out=not tokens_only
)
register_scale_out_api_routers(app, ("generate",))
response = TestClient(app).post(
GENERATE_PATH,
content="not JSON",
headers={"Content-Type": "application/json"},
)
assert response.status_code == 422
assert response.json()["detail"][0]["type"] == "json_invalid"
def test_scale_out_routes_can_be_enabled_for_generate_server():
paths = registered_paths(("generate",), enable_scale_out=True)
assert paths >= SCALE_OUT_PATHS
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
def test_abort_route_without_tokens_only_requires_api_key():
"""A decode pool that loads a tokenizer serves generate without
--tokens-only and still needs to abort, but only behind --api-key."""
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
assert ABORT_PATH in paths
assert ABORT_PATH.startswith(GUARDED_PREFIX)
assert UNAUTHENTICATED_ABORT_PATH not in paths
def test_unauthenticated_abort_route_is_not_registered_twice():
"""A pre-existing /abort_requests route must not be shadowed by a second
registration from the scale-out router."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@app.post(UNAUTHENTICATED_ABORT_PATH)
async def existing_abort_requests():
return None
attach_router(app)
paths = [route.path for route in app.routes]
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
assert ABORT_PATH in paths
@@ -1,419 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Roundtrip tests for multimodal serde used by the
token_in_token_out generate endpoint.
"""
import pytest
import torch
from pydantic import ValidationError
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
merge_mm_kwargs_items,
mm_kwargs_from_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.inputs import mm_input
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalFlatField,
MultiModalKwargsItem,
MultiModalSharedField,
PlaceholderRange,
)
def test_mm_kwargs_item_roundtrip():
"""Full roundtrip test with all three field types and multiple dtypes."""
e1 = MultiModalFieldElem(
data=torch.zeros(1000, dtype=torch.bfloat16),
field=MultiModalBatchedField(),
)
e2 = MultiModalFieldElem(
data=torch.ones(100, dtype=torch.int32),
field=MultiModalSharedField(batch_size=4),
)
e3 = MultiModalFieldElem(
data=torch.randn(20, dtype=torch.float32),
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
)
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
encoded = encode_mm_kwargs_item(item)
# Encoded result is a base64 string
assert isinstance(encoded, str)
decoded = decode_mm_kwargs_item(encoded)
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
def test_mm_kwargs_item_none_data():
"""Roundtrip with None data field."""
elem = MultiModalFieldElem(
data=None,
field=MultiModalSharedField(batch_size=2),
)
item = MultiModalKwargsItem({"empty": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
assert decoded["empty"].data is None
assert isinstance(decoded["empty"].field, MultiModalSharedField)
def test_mm_kwargs_item_nested_tensors():
"""Roundtrip with nested tensor data."""
nested = [torch.randn(3, 4), torch.randn(5, 4)]
elem = MultiModalFieldElem(
data=nested,
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"nested": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
decoded_data = decoded["nested"].data
assert len(decoded_data) == 2
assert torch.equal(nested[0], decoded_data[0])
assert torch.equal(nested[1], decoded_data[1])
def test_mm_features_with_kwargs_data():
"""Test that MultiModalFeatures can carry serialized tensor data."""
elem = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"pixel_values": elem})
encoded = encode_mm_kwargs_item(item)
features = MultiModalFeatures(
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
kwargs_data={"image": [encoded]},
)
# JSON roundtrip
json_str = features.model_dump_json()
features2 = MultiModalFeatures.model_validate_json(json_str)
assert features2.mm_hashes == {"image": ["abc123"]}
assert features2.kwargs_data is not None
assert len(features2.kwargs_data["image"]) == 1
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
assert torch.equal(elem.data, decoded["pixel_values"].data)
def _image_engine_input(
*,
pixel_keep_on_cpu: bool = False,
grid_keep_on_cpu: bool = True,
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3],
mm_kwargs={"image": [item]},
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
)
return engine_input, pixel_values, image_grid_thw
def test_render_extracts_metadata_fields_separately():
"""Render metadata contains CPU fields but excludes encoder inputs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
features = extract_mm_features(engine_input)
assert features is not None
assert features.mm_metadata is not None
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
assert features.kwargs_data is not None
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
assert set(full_data) == {"pixel_values", "image_grid_thw"}
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
def test_extract_without_kwargs_keeps_layout():
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
full = extract_mm_features(engine_input)
layout = extract_mm_features(engine_input, mm_payload="none")
assert full is not None and layout is not None
assert layout.kwargs_data is None
assert layout.mm_metadata is None
assert layout.mm_hashes == full.mm_hashes
assert layout.mm_placeholders == full.mm_placeholders
_MESSAGES = [{"role": "user", "content": "hi"}]
@pytest.mark.parametrize(
("request_cls", "body"),
[
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
(CompletionRequest, {"model": "m", "prompt": "hi"}),
(ResponsesRequest, {"model": "m", "input": "hi"}),
],
)
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
"""Clients of the deprecated bool keep getting layout-only responses."""
assert request_cls.model_validate(body).mm_payload == "tensors"
for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
request = request_cls.model_validate(
{**body, "return_mm_kwargs": return_mm_kwargs}
)
assert request.mm_payload == expected
explicit = request_cls.model_validate(
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
)
assert explicit.mm_payload == "tensors"
with pytest.raises(ValidationError):
request_cls.model_validate({**body, "mm_payload": "source"})
def test_extract_includes_declared_placeholder_metadata_fields():
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
grid_keep_on_cpu=False
)
features = extract_mm_features(
engine_input,
metadata_fields_for=lambda modality: (
{"image_grid_thw"} if modality == "image" else set()
),
)
assert features is not None
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_legacy_kwargs_only_generate_keeps_full_payload():
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
assert rendered.kwargs_data is not None
features = MultiModalFeatures(
mm_hashes=rendered.mm_hashes,
mm_placeholders=rendered.mm_placeholders,
kwargs_data=rendered.kwargs_data,
)
merged = mm_kwargs_from_features(features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_metadata_only_generate_requires_ec_transfer_params():
engine_input, _, _ = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": {
"mm_hashes": rendered.mm_hashes,
"mm_placeholders": {
modality: [p.model_dump() for p in ranges]
for modality, ranges in rendered.mm_placeholders.items()
},
"mm_metadata": rendered.mm_metadata,
},
}
try:
GenerateRequest.model_validate(payload)
raise AssertionError("expected metadata-only request without EC to fail")
except ValidationError as exc:
assert "ec_transfer_params" in str(exc)
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
request = GenerateRequest.model_validate(payload)
assert request.features is not None
assert request.features.kwargs_data is None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"image_grid_thw"}
@pytest.mark.parametrize("modality", ["image", "video"])
@pytest.mark.parametrize(
("has_metadata", "has_ec"),
[(True, False), (True, True), (False, False)],
ids=["metadata-without-ec", "metadata-with-ec", "cache-hit-without-ec"],
)
def test_mixed_items_require_ec_only_for_metadata_only(
modality: str, has_metadata: bool, has_ec: bool
):
"""A full item must not bypass the EC requirement for another item."""
engine_input, _, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
features = rendered.model_dump()
metadata = (
encode_mm_kwargs_item(
MultiModalKwargsItem({f"{modality}_grid_thw": image_grid_thw})
)
if has_metadata
else None
)
second_item = {
"mm_hashes": "second-item",
"mm_placeholders": {"offset": 2, "length": 1},
"kwargs_data": None,
"mm_metadata": metadata,
}
for key, value in second_item.items():
features[key].setdefault(modality, []).append(value)
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": features,
}
if has_ec:
payload["ec_transfer_params"] = {"second-item": {"peer_host": "10.0.0.1"}}
if has_metadata and not has_ec:
with pytest.raises(ValidationError, match="ec_transfer_params"):
GenerateRequest.model_validate(payload)
else:
GenerateRequest.model_validate(payload)
def test_kwargs_and_metadata_generate_does_not_require_ec():
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
request = GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": rendered.model_dump(),
}
)
assert request.features is not None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_cache_hit_null_kwargs_keeps_metadata_slot():
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3, 4],
mm_kwargs={"image": [item, None]},
mm_hashes={"image": ["hit", "cached"]},
mm_placeholders={
"image": [
PlaceholderRange(offset=0, length=1),
PlaceholderRange(offset=2, length=1),
]
},
)
features = extract_mm_features(engine_input)
assert features is not None
assert features.kwargs_data is not None
assert features.mm_metadata is not None
assert features.kwargs_data["image"][1] is None
assert features.mm_metadata["image"][1] is None
assert len(features.mm_metadata["image"]) == 2
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_features_alignment_rejects_length_mismatch():
try:
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
)
except ValidationError:
return
raise AssertionError("expected placeholder length mismatch to fail")
def test_metadata_can_replace_or_extend_full_mm_data():
"""Metadata-only requests retain the fields required by the model."""
metadata_elem = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
pixel_elem = MultiModalFieldElem(
data=torch.randn(5, 3),
field=MultiModalBatchedField(),
)
full_item = MultiModalKwargsItem(
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
)
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
merged = merge_mm_kwargs_items(full_item, metadata_item)
assert merged is not None
assert set(merged) == {"pixel_values", "image_grid_thw"}
@@ -1,266 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token_in_token_out request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and
fail loudly if the validator semantics ever drift.
"""
import json
import pytest
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
GenerateStreamResponse,
GenerateTextResponse,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.sampling_params import SamplingParams
def _base_payload() -> dict:
return {"token_ids": [1, 2, 3], "sampling_params": {}}
def test_omitted_max_tokens_is_not_provided():
"""Body without ``max_tokens`` must surface as 'not provided' so the
server can apply its own default instead of the dataclass 16."""
req = GenerateRequest.model_validate(_base_payload())
# SamplingParams' dataclass default leaks through the parsed instance —
# this is exactly the bug the server-side defaulting works around.
assert req.sampling_params.max_tokens == 16
assert not req.is_sampling_param_provided("max_tokens")
def test_explicit_max_tokens_is_provided():
"""Even when the client picks the same value as the dataclass default,
it must register as explicitly set so the server won't override it."""
payload = _base_payload()
payload["sampling_params"] = {"max_tokens": 16}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 16
assert req.is_sampling_param_provided("max_tokens")
payload["sampling_params"] = {"max_tokens": 256}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 256
assert req.is_sampling_param_provided("max_tokens")
def test_other_fields_tracked_independently():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.0}
req = GenerateRequest.model_validate(payload)
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_json_roundtrip_preserves_provided_keys():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.5}
req = GenerateRequest.model_validate_json(json.dumps(payload))
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_internal_instance_construction_treats_all_as_provided():
"""When internal callers build ``GenerateRequest`` from a pre-resolved
``SamplingParams`` instance, every field is considered explicitly set
so server-side defaulting can't clobber values resolved upstream."""
sp = SamplingParams(max_tokens=500, temperature=0.0)
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
assert req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
# And keys we never touched should also count as provided in this path.
assert req.is_sampling_param_provided("top_p")
def test_multimodal_features_reject_mismatched_parallel_fields():
with pytest.raises(ValueError, match="same length"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
kwargs_data={"image": ["encoded"]},
)
def test_multimodal_features_reject_overlapping_placeholders():
with pytest.raises(ValueError, match="non-overlapping"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={
"image": [
PlaceholderRangeInfo(offset=1, length=2),
PlaceholderRangeInfo(offset=2, length=2),
]
},
kwargs_data={"image": ["a", "b"]},
)
def test_generate_request_rejects_placeholder_outside_prompt():
with pytest.raises(ValueError, match="within the token_ids sequence"):
GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(),
features=MultiModalFeatures(
mm_hashes={"image": ["a"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
kwargs_data={"image": ["encoded"]},
),
)
def test_reasoning_parser_kwargs_rejects_unknown_keys():
"""The engine splats these into its reasoning parser's constructor, so a
client must not be able to pass arbitrary keyword arguments."""
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
with pytest.raises(ValidationError, match="tokenizer"):
GenerateRequest.model_validate(payload)
def test_rust_render_sampling_params_without_nulls_validate():
"""Rust render omits unset sampling fields; Python generate rejects nulls."""
sampling_params = {"max_tokens": 64, "watermarking": True}
request = GenerateRequest.model_validate(
{"token_ids": [1, 2, 3], "sampling_params": sampling_params}
)
assert request.sampling_params.max_tokens == 64
with pytest.raises(ValidationError, match="temperature"):
GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {**sampling_params, "temperature": None},
}
)
def test_output_mode_defaults_to_tokens():
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
def test_output_mode_text_is_accepted():
payload = {**_base_payload(), "output_mode": "text"}
assert GenerateRequest.model_validate(payload).output_mode == "text"
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
def test_output_mode_rejects_unsupported_levels(output_mode):
"""Levels the server does not implement fail instead of falling back to
tokens which would answer 200 with the wrong response shape."""
payload = {**_base_payload(), "output_mode": output_mode}
with pytest.raises(ValidationError, match="output_mode"):
GenerateRequest.model_validate(payload)
def test_text_mode_rejects_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "text",
}
with pytest.raises(ValidationError, match="detokenize"):
GenerateRequest.model_validate(payload)
def test_tokens_mode_allows_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "tokens",
}
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
def test_response_without_output_mode_parses_as_tokens():
"""Existing clients and older servers never send output_mode."""
parsed = TypeAdapter(GenerateResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensResponse)
assert parsed.output_mode == "tokens"
def test_stream_chunk_without_output_mode_parses_as_tokens():
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensStreamResponse)
def test_text_response_and_chunk_dispatch_on_output_mode():
response = TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
assert isinstance(response, GenerateTextResponse)
assert isinstance(chunk, GenerateTextStreamResponse)
def test_text_response_requires_text_on_every_choice():
with pytest.raises(ValidationError, match="text"):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0}]}
)
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
def test_response_rejects_unknown_output_mode(output_mode):
with pytest.raises(ValidationError):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": output_mode, "choices": []}
)
def test_derender_requests_accept_responses_without_output_mode():
DerenderChatRequest.model_validate(
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
)
DerenderChatStreamRequest.model_validate(
{
"stream": True,
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
}
)
def test_derender_requests_reject_text_responses():
"""Text responses are already detokenized with stop strings applied, so
re-decoding their token_ids would silently undo the truncation."""
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatRequest.model_validate({"generate_response": text})
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
def test_tokens_response_echoes_output_mode_without_text():
dumped = GenerateTokensResponse(
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
).model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for multimodal features through the /inference/v1/generate endpoint.
Mirrors test_serving_tokens.py but exercises the multimodal piping
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
/generate -> /detokenize path. Intentionally avoids running the HF
processor in the pytest parent process to keep os.fork() in other
tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free.
"""
import httpx
import pytest
import pytest_asyncio
from PIL import Image
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.utils import encode_image_url
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
GEN_ENDPOINT = "/inference/v1/generate"
RENDER_ENDPOINT = "/v1/chat/completions/render"
DETOKENIZE_ENDPOINT = "/detokenize"
@pytest.fixture(scope="module")
def test_image():
return Image.new("RGB", (224, 224), color=(255, 0, 0))
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
"--enable-scale-out",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_render_to_generate_roundtrip(client, test_image):
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
All preprocessing and detokenization happens in the server subprocess;
the pytest parent never imports transformers or touches torch tensors.
"""
data_url = encode_image_url(test_image, format="PNG")
render_payload = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{
"type": "text",
"text": "What color is this image? Answer in one word.",
},
],
}
],
}
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
render_resp.raise_for_status()
render_data = render_resp.json()
# Validate render output structure: keys exist and values are non-empty
# and well-typed.
assert "token_ids" in render_data
assert isinstance(render_data["token_ids"], list)
assert len(render_data["token_ids"]) > 0
assert all(isinstance(t, int) for t in render_data["token_ids"])
assert "features" in render_data
features = render_data["features"]
assert features is not None
assert isinstance(features, dict)
assert "mm_hashes" in features
assert "image" in features["mm_hashes"]
image_hashes = features["mm_hashes"]["image"]
assert isinstance(image_hashes, list)
assert len(image_hashes) > 0
assert all(isinstance(h, str) and h for h in image_hashes)
assert "mm_placeholders" in features
assert "image" in features["mm_placeholders"]
image_placeholders = features["mm_placeholders"]["image"]
assert isinstance(image_placeholders, list)
assert len(image_placeholders) > 0
for p in image_placeholders:
assert isinstance(p.get("offset"), int)
assert isinstance(p.get("length"), int)
assert p["length"] > 0
assert "kwargs_data" in features
assert "image" in features["kwargs_data"]
assert len(features["kwargs_data"]["image"]) > 0
# Build generate request from render output
generate_payload = render_data
generate_payload["sampling_params"] = {
"max_tokens": 10,
"temperature": 0.0,
}
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
assert isinstance(gen_data["choices"], list)
assert len(gen_data["choices"]) >= 1
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert isinstance(choice["token_ids"], list)
assert len(choice["token_ids"]) > 0
assert all(isinstance(t, int) for t in choice["token_ids"])
detok_resp = await client.post(
DETOKENIZE_ENDPOINT,
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
)
detok_resp.raise_for_status()
detok_data = detok_resp.json()
assert "prompt" in detok_data
text = detok_data["prompt"]
assert isinstance(text, str)
assert len(text) > 0
assert "red" in text.lower(), (
f"Expected model to identify the red image, got: {text!r}"
)
def _collapse_placeholders(
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
) -> list[int]:
"""Undo placeholder expansion: each expanded range becomes one token.
``/generate`` with ``content_parts`` expects unexpanded placeholders and
expands them itself, so the roundtrip must start from a collapsed prompt.
"""
ranges = sorted(
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
)
collapsed: list[int] = []
pos = 0
for offset, length in ranges:
collapsed.extend(token_ids[pos:offset])
collapsed.append(token_ids[offset])
pos = offset + length
collapsed.extend(token_ids[pos:])
return collapsed
async def _render_image_prompt(client, data_url: str) -> dict:
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What color is this?"},
],
}
],
},
)
render_resp.raise_for_status()
return render_resp.json()
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""Unexpanded prompt + content_parts must expand exactly like /render."""
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
assert len(unexpanded) < len(expanded)
gen_resp = await client.post(
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"return_token_ids": True,
},
)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
assert gen_data["prompt_token_ids"] == expanded
assert gen_data["mm_placeholders"] == mm_placeholders
@pytest.mark.asyncio
async def test_content_parts_streaming(client, test_image):
"""content_parts should work with streaming."""
import json as json_mod
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
async with client.stream(
"POST",
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
"stream": True,
"return_token_ids": True,
},
) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json_mod.loads(line[6:]))
assert len(chunks) > 0
prompt_metadata_chunks = [
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
]
assert len(prompt_metadata_chunks) == 1
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders
@@ -1,843 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
ABORT_ENDPOINT = "/inference/v1/abort_requests"
def get_vocab_size(model_name):
config = ModelConfig(
model=model_name,
seed=0,
dtype="bfloat16",
)
return config.get_vocab_size()
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(MODEL_NAME)
@pytest.fixture(scope="module")
def messages():
return [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "How many countries are in the EU?"},
]
@pytest.fixture(scope="module")
def server(request):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
# 1 ULP when the batch dimension (M) changes, because different M
# values cause the Tensile backend to select different tile
# configurations with different fp32 accumulation orders. With
# prefix caching, cache-miss prefills compute all tokens in one
# pass (large M) while cache-hit requests compute only the
# uncached suffix (small M), seeding a divergence that amplifies
# through the residual stream and flips argmax tokens.
# See: https://github.com/vllm-project/vllm/issues/33123
#
# Either disable prefix caching entirely, or enable it with
# --deterministic-prefix-caching which forces cache-miss prefills
# to split at block boundaries so the suffix GEMM shape is always
# identical regardless of cache state.
#
# Option A: disable prefix caching
"--no-enable-prefix-caching",
#
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
"--enable-scale-out",
]
extra_args = getattr(request, "param", None)
if extra_args is not None:
args = args + (
list(extra_args)
if isinstance(extra_args, (list, tuple))
else [str(extra_args)]
)
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_endpoint(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
assert data["choices"][0].get("sampling_mask") is None
@pytest.mark.asyncio
async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
"""Explicit null max_tokens must not skip the min_tokens bound."""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": None, "min_tokens": 2147483648},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "min_tokens" in resp.json()["error"]["message"]
followup = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=followup)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="sampling mask output is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"server",
[["--return-sampling-mask", "--logprobs-mode", "processed_logprobs"]],
indirect=True,
)
async def test_generate_sampling_mask(client):
top_k = 5
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.8,
"top_k": top_k,
"top_p": 0.9,
"ignore_eos": True,
"seed": 0,
"logprobs": top_k,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
choice = resp.json()["choices"][0]
token_ids = choice["token_ids"]
sampling_mask = choice["sampling_mask"]
assert sampling_mask is not None
assert len(token_ids) == len(sampling_mask)
vocab_size = get_vocab_size(MODEL_NAME)
logprobs_content = choice["logprobs"]["content"]
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
assert support
assert len(support) == len(set(support))
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
assert token_id in support
# processed_logprobs: exactly the support carries finite probability.
for top in entry["top_logprobs"]:
in_support = top["token_id"] in support
assert in_support == (top["logprob"] > -9999.0)
@pytest.mark.asyncio
async def test_generate_defaults_max_tokens_when_omitted(client):
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
defaulting was wired up, every request that didn't set ``max_tokens``
truncated mid-generation. The server should now fill it in from
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"temperature": 0.0,
"ignore_eos": True,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
completion_tokens = len(data["choices"][0]["token_ids"])
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
# should get ~1021 tokens of output (capped at max_model_len boundary).
assert completion_tokens > 16, (
f"expected server-side default to exceed the legacy 16-token cap, "
f"got {completion_tokens}"
)
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
)
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
"""Regression: ``n > 1`` must return ``n`` choices.
Non-streaming requests kept ``SamplingParams``' default output kind,
``CUMULATIVE``, so only the sequences updated during the last engine step
reached the response and the others were silently dropped.
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
@pytest.mark.asyncio
async def test_generate_stream(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": True,
}
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
payload_str = line[len("data: ") :]
if payload_str == "[DONE]":
break
chunks.append(json.loads(payload_str))
assert len(chunks) > 0
# Every chunk has choices with token_ids
all_token_ids = []
for chunk in chunks:
assert "choices" in chunk
assert len(chunk["choices"]) == 1
choice = chunk["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
all_token_ids.extend(choice["token_ids"])
# Last chunk should have a finish_reason
assert chunks[-1]["choices"][0]["finish_reason"] is not None
# Streaming should produce the same tokens as non-streaming
non_stream_resp = await client.post(
GEN_ENDPOINT,
json={
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
"stream": False,
},
)
non_stream_data = non_stream_resp.json()
# Just verify we got the right number of tokens
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
@pytest.mark.asyncio
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
async def test_generate_logprobs(client, logprobs_value):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.0,
"logprobs": logprobs_value,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["logprobs"] is not None
logprobs_content = choice["logprobs"]["content"]
assert len(logprobs_content) == len(choice["token_ids"])
for entry in logprobs_content:
assert "logprob" in entry
assert len(entry["top_logprobs"]) >= 1
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
for ignore_eos in [True, False]:
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
# NOTE coordinator will set this to skip detokenization
"detokenize": False,
"ignore_eos": ignore_eos,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
gen_token_ids = generate_data["choices"][0]["token_ids"]
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"ignore_eos": ignore_eos,
"chat_template_kwargs": {"enable_thinking": False},
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
if ignore_eos:
# When ignoring EOS, only compare up to the first EOS token
# Post-EOS generation is undefined and may differ
eos_tokens = {
tokenizer.eos_token_id,
*tokenizer.extra_special_tokens_ids,
}
# Find first EOS in generated tokens
eos_pos = None
for i, tid in enumerate(gen_token_ids):
if tid in eos_tokens:
eos_pos = i
break
if eos_pos is not None:
gen_token_ids_truncated = gen_token_ids[:eos_pos]
generate_res = tokenizer.decode(
gen_token_ids_truncated, skip_special_tokens=True
)
# Truncate completions_res to same length for comparison
completions_res = completions_res[: len(generate_res)]
assert generate_res == completions_res
@pytest.mark.asyncio
async def test_stop_string_workflow(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
# stop strings are only supported when detokenize is True.
"stop": ["27 member"],
},
# TODO stream test is much more interesting
"stream": False,
}
with pytest.raises(httpx.HTTPStatusError):
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_resp.raise_for_status()
payload["sampling_params"]["stop"] = None
generate_resp = await client.post(
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
# NOTE This is under the responsibility of the coordinator
# stop_checker = StopChecker(
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
# )
stop_str, truncate_to = check_stop_strings(
generate_res, len(generate_res), ["27 member"], False
)
assert stop_str == "27 member"
# abort request that hit stop string (requires tokens-only mode)
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
# res.raise_for_status()
generate_res = generate_res[:truncate_to]
# Get stop_str response from chat completions
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"stop": ["27 member"],
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_stop_strings_rejected_when_tokens_only(client):
"""--tokens-only forces detokenize=False after request validation, so
stop strings must be rejected explicitly rather than silently ignored."""
sampling_params = {"max_tokens": 5, "stop": ["never"]}
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": sampling_params,
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "stop strings" in resp.json()["error"]["message"]
sampling_params["stop"] = None
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server",
[
[
"--enable-lora",
"--lora-modules",
"Alice=charent/self_cognition_Alice",
"Bob=charent/self_cognition_Bob",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
],
indirect=True,
)
async def test_generate_with_lora_adapter(client, tokenizer, messages):
# Verify adapters are listed
models_resp = await client.get("/v1/models")
models_resp.raise_for_status()
models = {m["id"] for m in models_resp.json().get("data", [])}
assert {"Alice", "Bob"}.issubset(models)
# Generate using a LoRA adapter by specifying its name as the model
payload = {
"model": "Alice",
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": "Alice",
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
payload = {
"model": "Alice",
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
def _structured_chat_body(messages, **overrides) -> dict:
return {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 64,
"temperature": 0.0,
"chat_template_kwargs": {"enable_thinking": True},
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"count": {"type": "integer"}},
"required": ["count"],
},
},
},
**overrides,
}
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
render_resp = await client.post("/v1/chat/completions/render", json=body)
render_resp.raise_for_status()
generate_request = render_resp.json()
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
generate_resp.raise_for_status()
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
# <think> tokens, so the engine gates structured outputs differently when
# either reasoning field is dropped between render and generate.
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
"""With thinking on, structured outputs must wait for the reasoning to
end on generate, the same as on chat."""
body = _structured_chat_body(messages, return_token_ids=True)
chat_resp = await client.post("/v1/chat/completions", json=body)
chat_resp.raise_for_status()
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is False
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_ended(
client, tokenizer, messages
):
"""`include_reasoning=false` constrains from the first token on chat, so
generate must too, even though the engine's parser expects thinking."""
body = _structured_chat_body(messages, include_reasoning=False)
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is True
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
assert isinstance(output["count"], int)
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
return tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
def _text_mode_payload(token_ids, **sampling_params) -> dict:
return {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
"output_mode": "text",
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
resp.raise_for_status()
data = resp.json()
assert data["output_mode"] == "text"
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": {"enable_thinking": False},
}
chat_resp = await client.post("/v1/chat/completions", json=payload)
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
assert data["choices"][0]["text"] == chat_content
assert data["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
payload = _text_mode_payload(token_ids)
non_stream = await client.post(GEN_ENDPOINT, json=payload)
non_stream.raise_for_status()
chunks = []
async with client.stream(
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json.loads(line[len("data: ") :]))
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert chunks[-1]["choices"][0]["finish_reason"] is not None
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
assert streamed_text == non_stream.json()["choices"][0]["text"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"overrides, expected",
[
pytest.param(
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
),
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
],
)
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
payload = {
**_text_mode_payload([1, 2, 3], max_tokens=5),
**overrides,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert expected in resp.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_text_mode_rejected_when_tokens_only(client):
"""Without a tokenizer the detokenizer returns "", so the request must
fail instead of answering 200 with empty text."""
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
assert resp.status_code == 400
assert "requires a tokenizer" in resp.json()["error"]["message"]
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
payload["sampling_params"]["detokenize"] = False
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
async def test_stream_delivers_abort_finish_chunk(
client, tokenizer, messages, output_mode
):
"""The final output after an abort has no new token IDs, but the
stream must still deliver its finish reason."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"output_mode": output_mode,
"stream": True,
"request_id": f"{output_mode}-abort-e2e",
}
chunks = []
aborted = False
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
)
assert abort.status_code == 200
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
assert generated < 900
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
resp = await client.post(
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 200
resp = await client.post(ABORT_ENDPOINT, json={})
assert resp.status_code == 400
# The unauthenticated path is only served with --tokens-only.
resp = await client.post(
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 404
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from pydantic import ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own integer token id."""
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
)
token_ids = {e.token_id for e in result.content[0].top_logprobs}
assert token_ids == {262, 257}, f"got {token_ids}"
def test_sampled_entry_carries_token_id_and_rank():
"""The sampled token is identified by id, and the engine's rank survives."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
],
)
entry = result.content[0]
assert entry.token_id == 262
assert entry.logprob == -0.1
assert entry.rank == 1
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
# No tokenizer on the generate server: no string token, no bytes.
assert not hasattr(entry, "token")
assert not hasattr(entry, "bytes")
def test_sampled_token_absent_from_topk_uses_sentinel():
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[5],
top_logprobs=[{7: Logprob(-0.9)}],
)
entry = result.content[0]
assert entry.token_id == 5
assert entry.logprob == -9999.0
assert entry.rank is None
assert entry.top_logprobs == []
def test_logprobs_zero_emits_sampled_token():
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9)}],
)
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
def test_logprobs_minus_one_emits_all_tokens():
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
assert len(result.content[0].top_logprobs) == 2
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
entry = result.content[0]
assert entry.token == "token_id:7"
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
tokenizer=tokenizer,
)
entry = result.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
tokenizer.decode.assert_not_called()
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")
def test_sampled_token_outside_topk_comes_first():
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
all k + 1 candidates when the sampled token is outside the top k; derender
applies each endpoint's cut (#59513)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[50],
top_logprobs=[
{
50: Logprob(-3.0, rank=5),
10: Logprob(-0.2, rank=1),
20: Logprob(-1.0, rank=2),
}
],
)
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
def test_logprob_is_required_on_the_wire():
"""A payload missing logprob is rejected rather than read as -9999."""
with pytest.raises(ValidationError):
GenerateLogProb.model_validate({"token_id": 1})
def test_rank_zero_from_a_nan_logprob_is_none():
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
)
entry = result.content[0]
assert entry.rank is None
assert entry.top_logprobs[0].rank is None
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
def test_text_logprobs_clamp_a_nan_logprob():
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
tokenizer=MagicMock(),
)
entry = result.content[0]
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
@@ -120,7 +120,7 @@ class TestAffectedModulesUseSanitize:
@pytest.mark.parametrize(
"module",
[
"vllm.entrypoints.speech_to_text.realtime.connection",
"vllm.entrypoints.serve.exception_handling.handlers.http",
],
)
def test_module_calls_sanitize_message(self, module: str):
@@ -1,371 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
import shutil
from contextlib import suppress
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
BADREQUEST_CASES = [
(
"test_rank",
{"r": 1024},
"is greater than max_lora_rank",
),
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
(
"test_modules_to_save",
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
]
@pytest.fixture(scope="module", params=[True])
def server_with_lora_modules_json(request, qwen3_lora_files):
# Define the json format LoRA module configurations
lora_module_1 = {
"name": "qwen3-lora",
"path": qwen3_lora_files,
"base_model_name": MODEL_NAME,
}
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
json.dumps(lora_module_1),
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"64",
]
# Enable the /v1/load_lora_adapter endpoint
envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server_with_lora_modules_json):
async with server_with_lora_modules_json.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert served_model.parent is None
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
@pytest.mark.asyncio
async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
)
# Ensure adapter loads before querying /models
assert "success" in response
models = await client.models.list()
models = models.data
dynamic_lora_model = models[-1]
assert dynamic_lora_model.root == qwen3_lora_files
assert dynamic_lora_model.parent == MODEL_NAME
assert dynamic_lora_model.id == "qwen3-lora-3"
@pytest.mark.asyncio
async def test_load_lora_adapter_with_same_name_replaces_inplace(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
):
"""Test that loading a LoRA adapter with the same name replaces it inplace."""
adapter_name = "replaceable-adapter"
messages = [
{"content": "Follow the instructions to make animal noises", "role": "system"},
{"content": "Make your favorite animal noise.", "role": "user"},
]
# Load LoRA that makes model meow
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Meow Meow Meow" in completion.choices[0].message.content
# Load LoRA that makes model woof
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_woofing_lora_files,
"load_inplace": True,
},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Woof Woof Woof" in completion.choices[0].message.content
@pytest.mark.asyncio
async def test_load_lora_adapter_with_load_inplace_false_errors(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files
):
"""Test that load_inplace=False returns an error when adapter already exists."""
adapter_name = "test-load-inplace-false"
# Load LoRA adapter first time (should succeed)
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
# Try to load the same adapter again with load_inplace=False (should fail)
with pytest.raises(openai.BadRequestError) as exc_info:
await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_meowing_lora_files,
},
)
# Verify the error message
assert "already been loaded" in str(exc_info.value)
@pytest.mark.asyncio
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
with pytest.raises(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
@pytest.mark.asyncio
async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
with pytest.raises(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid-json", "lora_path": str(invalid_files)},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES)
async def test_dynamic_lora_badrequests(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
test_name: str,
config_change: dict,
expected_error: str,
):
# Create test directory
test_dir = tmp_path / test_name
# Copy adapter files
shutil.copytree(qwen3_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(config_change)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
# Test loading the adapter
with pytest.raises(openai.InternalServerError, match=expected_error):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": test_name, "lora_path": str(test_dir)},
)
@pytest.mark.asyncio
async def test_multiple_lora_adapters(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
"""Validate that many loras can be dynamically registered and inferenced
with concurrently"""
# This test file configures the server with --max-cpu-loras=2 and this test
# will concurrently load 10 adapters, so it should flex the LRU cache
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
lora_tasks = []
for i in range(10):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
@pytest.mark.asyncio
async def test_loading_invalid_adapters_does_not_break_others(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
stop_good_requests_event = asyncio.Event()
async def run_good_requests(client):
# Run chat completions requests until event set
results = []
while not stop_good_requests_event.is_set():
try:
batch = await client.completions.create(
model="qwen3-lora",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
results.append(batch)
except Exception as e:
results.append(e)
return results
# Create task to run good requests
good_task = asyncio.create_task(run_good_requests(client))
# Run a bunch of bad adapter loads
for _ in range(25):
with suppress(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
for _ in range(25):
with suppress(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid", "lora_path": str(invalid_files)},
)
# Ensure all the running requests with lora adapters succeeded
stop_good_requests_event.set()
results = await good_task
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
# Ensure we can load another adapter and run it
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "valid", "lora_path": qwen3_lora_files},
)
await client.completions.create(
model="valid",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
@pytest.mark.asyncio
async def test_beam_search_with_lora_adapters(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
):
"""Validate that async beam search can be used with lora."""
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
extra_body=dict(use_beam_search=True),
)
lora_tasks = []
for i in range(3):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
@@ -214,7 +214,6 @@ def test_request_logger_log_outputs_integration():
prompt_token_ids=[1, 2, 3],
prompt_embeds=None,
params=None,
lora_request=None,
)
request_logger.log_outputs(
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.audio import AudioAsset
def add_attention_backend(server_args, attention_config):
"""Append attention backend CLI arg if specified.
Args:
server_args: List of server arguments to extend in-place.
attention_config: Dict with 'backend' key, or None.
"""
if attention_config and "backend" in attention_config:
server_args.extend(["--attention-backend", attention_config["backend"]])
@pytest.fixture
def mary_had_lamb():
path = AudioAsset("mary_had_lamb").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def winning_call():
path = AudioAsset("winning_call").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def foscolo():
# Test translation it->en
path = AudioAsset("azacinto_foscolo").get_local_path()
with open(str(path), "rb") as f:
yield f
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
@pytest.fixture(scope="module")
def transcription_server_with_force_include_usage():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-num-seqs",
"4",
"--enforce-eager",
"--enable-force-include-usage",
"--gpu-memory-utilization",
"0.2",
]
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def transcription_client_with_force_include_usage(
transcription_server_with_force_include_usage,
):
async with (
transcription_server_with_force_include_usage.get_async_client() as async_client
):
yield async_client
@pytest.mark.asyncio
async def test_transcription_with_enable_force_include_usage(
transcription_client_with_force_include_usage, winning_call
):
res = (
await transcription_client_with_force_include_usage.audio.transcriptions.create(
model="openai/whisper-large-v3-turbo",
file=winning_call,
language="en",
temperature=0.0,
stream=True,
timeout=30,
)
)
async for chunk in res:
if not len(chunk.choices):
# final usage sent
usage = chunk.usage
assert isinstance(usage, dict)
assert usage["prompt_tokens"] > 0
assert usage["completion_tokens"] > 0
assert usage["total_tokens"] > 0
else:
assert not hasattr(chunk, "usage")