[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -65,37 +65,3 @@ def sample_structured_outputs_choices():
|
||||
"Swift",
|
||||
"Kotlin",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_meowing_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_woofing_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def opt125_lora_files() -> str:
|
||||
"""Download opt-125m LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="peft-internal-testing/opt-125m-dummy-lora"
|
||||
)
|
||||
|
||||
@@ -1,28 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
LORA_MODULE = {
|
||||
"name": "module2",
|
||||
"path": "/path/to/module2",
|
||||
"base_model_name": "llama",
|
||||
}
|
||||
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
assert CHATML_JINJA_PATH.exists()
|
||||
|
||||
|
||||
def _build_vllm_parsers():
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
@@ -96,87 +84,6 @@ def test_logging_config_cli_args(serve_parser):
|
||||
assert config.pylogging_config_file == "/tmp/flat.json"
|
||||
|
||||
|
||||
### Tests for LoRA module parsing
|
||||
def test_valid_key_value_format(serve_parser):
|
||||
# Test old format: name=path
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
]
|
||||
)
|
||||
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_valid_json_format(serve_parser):
|
||||
# Test valid JSON format input
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_invalid_json_format(serve_parser):
|
||||
# Test invalid JSON format input, missing closing brace
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_type_error(serve_parser):
|
||||
# Test type error when values are not JSON or key=value
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"invalid_format", # This is not JSON or key=value format
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_json_field(serve_parser):
|
||||
# Test valid JSON format but missing required fields
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
'{"name": "module4"}', # Missing required 'path' field
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_empty_values(serve_parser):
|
||||
# Test when no LoRA modules are provided
|
||||
args = serve_parser.parse_args(["--lora-modules", ""])
|
||||
assert args.lora_modules == []
|
||||
|
||||
|
||||
def test_multiple_valid_inputs(serve_parser):
|
||||
# Test multiple valid inputs (both old and JSON format)
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module1", path="/path/to/module1"),
|
||||
LoRAModulePath(
|
||||
name="module2", path="/path/to/module2", base_model_name="llama"
|
||||
),
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
|
||||
|
||||
@@ -241,10 +148,12 @@ def test_reasoning_parser_none_disables_parser(serve_parser, value, expected):
|
||||
assert args.reasoning_parser == expected
|
||||
|
||||
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser):
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser, tmp_path):
|
||||
"""Ensure validation passes if the chat template exists."""
|
||||
chat_template = tmp_path / "template.jinja"
|
||||
chat_template.write_text("{{ messages }}")
|
||||
args = serve_parser.parse_args(
|
||||
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
|
||||
args=["--chat-template", chat_template.absolute().as_posix()]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
@@ -389,32 +298,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||
else:
|
||||
with pytest.raises(raises):
|
||||
vllm_parser.parse_args(args=args)
|
||||
|
||||
|
||||
### Tests for LoRA target modules parsing
|
||||
def test_lora_target_modules_single(serve_parser):
|
||||
"""Test parsing single lora-target-modules argument."""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--enable-lora", "--lora-target-modules", "o_proj"]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_multiple(serve_parser):
|
||||
"""Test parsing multiple lora-target-modules arguments."""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-lora",
|
||||
"--lora-target-modules",
|
||||
"o_proj",
|
||||
"qkv_proj",
|
||||
"down_proj",
|
||||
]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_default_none(serve_parser):
|
||||
"""Test that lora-target-modules defaults to None."""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.lora_target_modules is None
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.conftest import AudioTestAssets
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
# NOTE - the tests in this module are currently analogous to test_chat, but are
|
||||
# separated to avoid OOM killing due to module-scoped servers, since we
|
||||
# need a multimodal model for these tests.
|
||||
|
||||
# Contains a modality specific lora alongside the base model
|
||||
MULTIMODAL_MODEL_NAME = hf_api().snapshot_download(
|
||||
"microsoft/Phi-4-multimodal-instruct"
|
||||
)
|
||||
AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
|
||||
|
||||
ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def multimodal_server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"half",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"speech={AUDIO_LORA_PATH}",
|
||||
"--max-lora-rank",
|
||||
"320",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
"--trust-remote-code",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
"--default-mm-loras",
|
||||
f'{{"audio": "{AUDIO_LORA_PATH}"}}',
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MULTIMODAL_MODEL_NAME, args, max_wait_seconds=480
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def multi_modal_client(multimodal_server):
|
||||
async with multimodal_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# base model with default lora should give the same response as lora model
|
||||
"model_name",
|
||||
[MULTIMODAL_MODEL_NAME, "speech"],
|
||||
)
|
||||
async def test_default_mm_lora_chat_completions(
|
||||
model_name: str,
|
||||
multi_modal_client: openai.AsyncOpenAI,
|
||||
audio_assets: AudioTestAssets,
|
||||
):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Can you transcribe this audio?",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_assets[0].url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
chat_completion = await multi_modal_client.chat.completions.create(
|
||||
model=model_name, messages=messages, max_completion_tokens=128, temperature=0.0
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) > 0
|
||||
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 0
|
||||
assert message.content == ACTIVE_MM_LORA_RESPONSE
|
||||
@@ -27,15 +27,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def zephyr_lora_files():
|
||||
"""Download zephyr LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(zephyr_lora_files):
|
||||
def server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
@@ -43,14 +35,6 @@ def server(zephyr_lora_files):
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"zephyr-lora={zephyr_lora_files}",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
@@ -67,9 +51,8 @@ async def client(server):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# first test base model, then test loras
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -91,9 +74,8 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# just test 1 lora hereafter
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -119,7 +101,7 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -145,7 +127,7 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -254,7 +236,7 @@ async def test_more_than_one_prompt_logprobs_chat(
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -297,9 +279,8 @@ async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# just test 1 lora hereafter
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -345,7 +326,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
|
||||
["HuggingFaceH4/zephyr-7b-beta"],
|
||||
)
|
||||
async def test_chat_completion_stream_options(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
|
||||
@@ -9,7 +9,6 @@ import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
# downloading lora to test lora requests
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
|
||||
@@ -9,12 +9,10 @@ from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
|
||||
# generation quality here
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(qwen3_lora_files):
|
||||
def server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
@@ -22,14 +20,6 @@ def server(qwen3_lora_files):
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"qwen3-lora={qwen3_lora_files}",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
@@ -45,12 +35,9 @@ async def client(server):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
async def test_check_models(client: openai.AsyncOpenAI):
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
served_model = models[0]
|
||||
lora_models = models[1:]
|
||||
assert served_model.id == MODEL_NAME
|
||||
assert served_model.root == MODEL_NAME
|
||||
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
|
||||
assert lora_models[0].id == "qwen3-lora"
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the token-offsets request/response protocol wiring:
|
||||
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
|
||||
serialization boundary. Plain Pydantic field storage is not retested here.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _model_config() -> Mock:
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.max_model_len = 128
|
||||
return model_config
|
||||
|
||||
|
||||
def test_completion_flag_forwarded_to_tok_params():
|
||||
"""build_tok_params must forward return_token_offsets, defaulting to
|
||||
False (zero behavioral change for existing callers) and coercing JSON
|
||||
null to False via the bool() guard."""
|
||||
cfg = _model_config()
|
||||
|
||||
default = CompletionRequest(model="m", prompt="hi")
|
||||
assert default.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
on = CompletionRequest(model="m", prompt="hi", return_token_offsets=True)
|
||||
assert on.build_tok_params(cfg).return_token_offsets is True
|
||||
|
||||
null = CompletionRequest(model="m", prompt="hi", return_token_offsets=None)
|
||||
assert null.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
|
||||
def test_chat_flag_forwarded_to_tok_params():
|
||||
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
|
||||
its return_token_offsets forwarding is verified independently."""
|
||||
cfg = _model_config()
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
default = ChatCompletionRequest(model="m", messages=messages)
|
||||
assert default.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
on = ChatCompletionRequest(model="m", messages=messages, return_token_offsets=True)
|
||||
assert on.build_tok_params(cfg).return_token_offsets is True
|
||||
|
||||
null = ChatCompletionRequest(
|
||||
model="m", messages=messages, return_token_offsets=None
|
||||
)
|
||||
assert null.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
|
||||
def test_generate_request_token_offsets_default_none():
|
||||
"""Defaults to None so existing /v1/.../render responses are unchanged."""
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=SamplingParams())
|
||||
assert req.token_offsets is None
|
||||
|
||||
|
||||
def test_generate_request_token_offsets_survive_json_round_trip():
|
||||
"""GenerateRequest crosses the disagg serialization boundary; the
|
||||
tuple[int, int] offsets must survive model_dump and re-validate."""
|
||||
req = GenerateRequest(
|
||||
token_ids=[10, 20],
|
||||
sampling_params=SamplingParams(),
|
||||
token_offsets=[(0, 1), (1, 3)],
|
||||
)
|
||||
dumped = req.model_dump()
|
||||
assert dumped["token_offsets"] == [(0, 1), (1, 3)]
|
||||
# Re-validate from the dumped dict (sampling_params doesn't round-trip
|
||||
# cleanly via dump, so re-inject a fresh instance).
|
||||
again = GenerateRequest.model_validate(
|
||||
{**dumped, "sampling_params": SamplingParams()}
|
||||
)
|
||||
assert again.token_offsets == [(0, 1), (1, 3)]
|
||||
@@ -1,125 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
|
||||
from vllm.entrypoints.serve.middleware.authenticate import (
|
||||
GUARDED_PREFIX,
|
||||
AuthenticationMiddleware,
|
||||
)
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
ABORT_PATH = "/inference/v1/abort_requests"
|
||||
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
|
||||
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
*,
|
||||
tokens_only: bool = False,
|
||||
enable_scale_out: bool = False,
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(
|
||||
tokens_only=tokens_only,
|
||||
enable_scale_out=enable_scale_out,
|
||||
enable_fault_tolerance=False,
|
||||
)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
|
||||
@pytest.mark.parametrize("authorized", [False, True])
|
||||
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
|
||||
app = FastAPI()
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
|
||||
headers = {"Authorization": "Bearer secret"} if authorized else {}
|
||||
client = TestClient(app)
|
||||
response = client.post(path, content="not JSON", headers=headers)
|
||||
|
||||
assert response.status_code == (403 if authorized else 401)
|
||||
if authorized:
|
||||
assert "--enable-scale-out" in response.json()["error"]["message"]
|
||||
assert response.json()["error"]["type"] == "PermissionDeniedError"
|
||||
assert path not in app.openapi()["paths"]
|
||||
assert client.post("/unknown", headers=headers).status_code == 404
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert [
|
||||
record.levelno
|
||||
for record in caplog.records
|
||||
if "--enable-scale-out" in record.message
|
||||
] == [logging.INFO]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens_only", [False, True])
|
||||
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(
|
||||
tokens_only=tokens_only, enable_scale_out=not tokens_only
|
||||
)
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
response = TestClient(app).post(
|
||||
GENERATE_PATH,
|
||||
content="not JSON",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"][0]["type"] == "json_invalid"
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
paths = registered_paths(("generate",), enable_scale_out=True)
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
|
||||
|
||||
|
||||
def test_abort_route_without_tokens_only_requires_api_key():
|
||||
"""A decode pool that loads a tokenizer serves generate without
|
||||
--tokens-only and still needs to abort, but only behind --api-key."""
|
||||
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
|
||||
|
||||
assert ABORT_PATH in paths
|
||||
assert ABORT_PATH.startswith(GUARDED_PREFIX)
|
||||
assert UNAUTHENTICATED_ABORT_PATH not in paths
|
||||
|
||||
|
||||
def test_unauthenticated_abort_route_is_not_registered_twice():
|
||||
"""A pre-existing /abort_requests route must not be shadowed by a second
|
||||
registration from the scale-out router."""
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(tokens_only=True)
|
||||
|
||||
@app.post(UNAUTHENTICATED_ABORT_PATH)
|
||||
async def existing_abort_requests():
|
||||
return None
|
||||
|
||||
attach_router(app)
|
||||
|
||||
paths = [route.path for route in app.routes]
|
||||
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
|
||||
assert ABORT_PATH in paths
|
||||
@@ -1,419 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Roundtrip tests for multimodal serde used by the
|
||||
token_in_token_out generate endpoint.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
extract_mm_features,
|
||||
merge_mm_kwargs_items,
|
||||
mm_kwargs_from_features,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
decode_mm_kwargs_item,
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.inputs import mm_input
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFieldElem,
|
||||
MultiModalFlatField,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_roundtrip():
|
||||
"""Full roundtrip test with all three field types and multiple dtypes."""
|
||||
e1 = MultiModalFieldElem(
|
||||
data=torch.zeros(1000, dtype=torch.bfloat16),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
e2 = MultiModalFieldElem(
|
||||
data=torch.ones(100, dtype=torch.int32),
|
||||
field=MultiModalSharedField(batch_size=4),
|
||||
)
|
||||
e3 = MultiModalFieldElem(
|
||||
data=torch.randn(20, dtype=torch.float32),
|
||||
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
|
||||
)
|
||||
|
||||
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
# Encoded result is a base64 string
|
||||
assert isinstance(encoded, str)
|
||||
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
|
||||
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
|
||||
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
|
||||
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
|
||||
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
|
||||
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
|
||||
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_none_data():
|
||||
"""Roundtrip with None data field."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=None,
|
||||
field=MultiModalSharedField(batch_size=2),
|
||||
)
|
||||
item = MultiModalKwargsItem({"empty": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert decoded["empty"].data is None
|
||||
assert isinstance(decoded["empty"].field, MultiModalSharedField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_nested_tensors():
|
||||
"""Roundtrip with nested tensor data."""
|
||||
nested = [torch.randn(3, 4), torch.randn(5, 4)]
|
||||
elem = MultiModalFieldElem(
|
||||
data=nested,
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"nested": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
decoded_data = decoded["nested"].data
|
||||
assert len(decoded_data) == 2
|
||||
assert torch.equal(nested[0], decoded_data[0])
|
||||
assert torch.equal(nested[1], decoded_data[1])
|
||||
|
||||
|
||||
def test_mm_features_with_kwargs_data():
|
||||
"""Test that MultiModalFeatures can carry serialized tensor data."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"pixel_values": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
features = MultiModalFeatures(
|
||||
mm_hashes={"image": ["abc123"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
|
||||
kwargs_data={"image": [encoded]},
|
||||
)
|
||||
|
||||
# JSON roundtrip
|
||||
json_str = features.model_dump_json()
|
||||
features2 = MultiModalFeatures.model_validate_json(json_str)
|
||||
|
||||
assert features2.mm_hashes == {"image": ["abc123"]}
|
||||
assert features2.kwargs_data is not None
|
||||
assert len(features2.kwargs_data["image"]) == 1
|
||||
|
||||
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
|
||||
assert torch.equal(elem.data, decoded["pixel_values"].data)
|
||||
|
||||
|
||||
def _image_engine_input(
|
||||
*,
|
||||
pixel_keep_on_cpu: bool = False,
|
||||
grid_keep_on_cpu: bool = True,
|
||||
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
|
||||
pixel_values = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
|
||||
)
|
||||
image_grid_thw = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
|
||||
)
|
||||
item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
|
||||
)
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_kwargs={"image": [item]},
|
||||
mm_hashes={"image": ["abc123"]},
|
||||
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
|
||||
)
|
||||
return engine_input, pixel_values, image_grid_thw
|
||||
|
||||
|
||||
def test_render_extracts_metadata_fields_separately():
|
||||
"""Render metadata contains CPU fields but excludes encoder inputs."""
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
|
||||
features = extract_mm_features(engine_input)
|
||||
|
||||
assert features is not None
|
||||
assert features.mm_metadata is not None
|
||||
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
|
||||
assert features.mm_metadata is not None
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert set(metadata) == {"image_grid_thw"}
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
assert features.kwargs_data is not None
|
||||
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
|
||||
assert set(full_data) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
|
||||
|
||||
|
||||
def test_extract_without_kwargs_keeps_layout():
|
||||
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
|
||||
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
|
||||
|
||||
full = extract_mm_features(engine_input)
|
||||
layout = extract_mm_features(engine_input, mm_payload="none")
|
||||
|
||||
assert full is not None and layout is not None
|
||||
assert layout.kwargs_data is None
|
||||
assert layout.mm_metadata is None
|
||||
assert layout.mm_hashes == full.mm_hashes
|
||||
assert layout.mm_placeholders == full.mm_placeholders
|
||||
|
||||
|
||||
_MESSAGES = [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_cls", "body"),
|
||||
[
|
||||
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
|
||||
(CompletionRequest, {"model": "m", "prompt": "hi"}),
|
||||
(ResponsesRequest, {"model": "m", "input": "hi"}),
|
||||
],
|
||||
)
|
||||
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
|
||||
"""Clients of the deprecated bool keep getting layout-only responses."""
|
||||
assert request_cls.model_validate(body).mm_payload == "tensors"
|
||||
|
||||
for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
|
||||
request = request_cls.model_validate(
|
||||
{**body, "return_mm_kwargs": return_mm_kwargs}
|
||||
)
|
||||
assert request.mm_payload == expected
|
||||
|
||||
explicit = request_cls.model_validate(
|
||||
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
|
||||
)
|
||||
assert explicit.mm_payload == "tensors"
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
request_cls.model_validate({**body, "mm_payload": "source"})
|
||||
|
||||
|
||||
def test_extract_includes_declared_placeholder_metadata_fields():
|
||||
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
|
||||
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
|
||||
grid_keep_on_cpu=False
|
||||
)
|
||||
|
||||
features = extract_mm_features(
|
||||
engine_input,
|
||||
metadata_fields_for=lambda modality: (
|
||||
{"image_grid_thw"} if modality == "image" else set()
|
||||
),
|
||||
)
|
||||
|
||||
assert features is not None
|
||||
assert features.mm_metadata is not None
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert set(metadata) == {"image_grid_thw"}
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_legacy_kwargs_only_generate_keeps_full_payload():
|
||||
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
assert rendered.kwargs_data is not None
|
||||
|
||||
features = MultiModalFeatures(
|
||||
mm_hashes=rendered.mm_hashes,
|
||||
mm_placeholders=rendered.mm_placeholders,
|
||||
kwargs_data=rendered.kwargs_data,
|
||||
)
|
||||
merged = mm_kwargs_from_features(features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(item["pixel_values"].data, pixel_values.data)
|
||||
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_metadata_only_generate_requires_ec_transfer_params():
|
||||
engine_input, _, _ = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": {
|
||||
"mm_hashes": rendered.mm_hashes,
|
||||
"mm_placeholders": {
|
||||
modality: [p.model_dump() for p in ranges]
|
||||
for modality, ranges in rendered.mm_placeholders.items()
|
||||
},
|
||||
"mm_metadata": rendered.mm_metadata,
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
GenerateRequest.model_validate(payload)
|
||||
raise AssertionError("expected metadata-only request without EC to fail")
|
||||
except ValidationError as exc:
|
||||
assert "ec_transfer_params" in str(exc)
|
||||
|
||||
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
|
||||
request = GenerateRequest.model_validate(payload)
|
||||
assert request.features is not None
|
||||
assert request.features.kwargs_data is None
|
||||
merged = mm_kwargs_from_features(request.features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"image_grid_thw"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modality", ["image", "video"])
|
||||
@pytest.mark.parametrize(
|
||||
("has_metadata", "has_ec"),
|
||||
[(True, False), (True, True), (False, False)],
|
||||
ids=["metadata-without-ec", "metadata-with-ec", "cache-hit-without-ec"],
|
||||
)
|
||||
def test_mixed_items_require_ec_only_for_metadata_only(
|
||||
modality: str, has_metadata: bool, has_ec: bool
|
||||
):
|
||||
"""A full item must not bypass the EC requirement for another item."""
|
||||
engine_input, _, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
features = rendered.model_dump()
|
||||
metadata = (
|
||||
encode_mm_kwargs_item(
|
||||
MultiModalKwargsItem({f"{modality}_grid_thw": image_grid_thw})
|
||||
)
|
||||
if has_metadata
|
||||
else None
|
||||
)
|
||||
second_item = {
|
||||
"mm_hashes": "second-item",
|
||||
"mm_placeholders": {"offset": 2, "length": 1},
|
||||
"kwargs_data": None,
|
||||
"mm_metadata": metadata,
|
||||
}
|
||||
for key, value in second_item.items():
|
||||
features[key].setdefault(modality, []).append(value)
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": features,
|
||||
}
|
||||
if has_ec:
|
||||
payload["ec_transfer_params"] = {"second-item": {"peer_host": "10.0.0.1"}}
|
||||
|
||||
if has_metadata and not has_ec:
|
||||
with pytest.raises(ValidationError, match="ec_transfer_params"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
else:
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_kwargs_and_metadata_generate_does_not_require_ec():
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
request = GenerateRequest.model_validate(
|
||||
{
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": rendered.model_dump(),
|
||||
}
|
||||
)
|
||||
assert request.features is not None
|
||||
merged = mm_kwargs_from_features(request.features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(item["pixel_values"].data, pixel_values.data)
|
||||
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_cache_hit_null_kwargs_keeps_metadata_slot():
|
||||
pixel_values = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
image_grid_thw = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=True),
|
||||
)
|
||||
item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
|
||||
)
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=[1, 2, 3, 4],
|
||||
mm_kwargs={"image": [item, None]},
|
||||
mm_hashes={"image": ["hit", "cached"]},
|
||||
mm_placeholders={
|
||||
"image": [
|
||||
PlaceholderRange(offset=0, length=1),
|
||||
PlaceholderRange(offset=2, length=1),
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
features = extract_mm_features(engine_input)
|
||||
assert features is not None
|
||||
assert features.kwargs_data is not None
|
||||
assert features.mm_metadata is not None
|
||||
assert features.kwargs_data["image"][1] is None
|
||||
assert features.mm_metadata["image"][1] is None
|
||||
assert len(features.mm_metadata["image"]) == 2
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_features_alignment_rejects_length_mismatch():
|
||||
try:
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
)
|
||||
except ValidationError:
|
||||
return
|
||||
raise AssertionError("expected placeholder length mismatch to fail")
|
||||
|
||||
|
||||
def test_metadata_can_replace_or_extend_full_mm_data():
|
||||
"""Metadata-only requests retain the fields required by the model."""
|
||||
metadata_elem = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=True),
|
||||
)
|
||||
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
|
||||
pixel_elem = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
full_item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
|
||||
)
|
||||
|
||||
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
|
||||
merged = merge_mm_kwargs_items(full_item, metadata_item)
|
||||
assert merged is not None
|
||||
assert set(merged) == {"pixel_values", "image_grid_thw"}
|
||||
@@ -1,266 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the token_in_token_out request/response protocol.
|
||||
|
||||
These tests intentionally avoid spinning up a server — they exercise the
|
||||
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
||||
fail loudly if the validator semantics ever drift.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateStreamResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _base_payload() -> dict:
|
||||
return {"token_ids": [1, 2, 3], "sampling_params": {}}
|
||||
|
||||
|
||||
def test_omitted_max_tokens_is_not_provided():
|
||||
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
||||
server can apply its own default instead of the dataclass 16."""
|
||||
req = GenerateRequest.model_validate(_base_payload())
|
||||
# SamplingParams' dataclass default leaks through the parsed instance —
|
||||
# this is exactly the bug the server-side defaulting works around.
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_explicit_max_tokens_is_provided():
|
||||
"""Even when the client picks the same value as the dataclass default,
|
||||
it must register as explicitly set so the server won't override it."""
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"max_tokens": 16}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
payload["sampling_params"] = {"max_tokens": 256}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 256
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_other_fields_tracked_independently():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.0}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_json_roundtrip_preserves_provided_keys():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.5}
|
||||
req = GenerateRequest.model_validate_json(json.dumps(payload))
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_internal_instance_construction_treats_all_as_provided():
|
||||
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
||||
``SamplingParams`` instance, every field is considered explicitly set
|
||||
so server-side defaulting can't clobber values resolved upstream."""
|
||||
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
# And keys we never touched should also count as provided in this path.
|
||||
assert req.is_sampling_param_provided("top_p")
|
||||
|
||||
|
||||
def test_multimodal_features_reject_mismatched_parallel_fields():
|
||||
with pytest.raises(ValueError, match="same length"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
)
|
||||
|
||||
|
||||
def test_multimodal_features_reject_overlapping_placeholders():
|
||||
with pytest.raises(ValueError, match="non-overlapping"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={
|
||||
"image": [
|
||||
PlaceholderRangeInfo(offset=1, length=2),
|
||||
PlaceholderRangeInfo(offset=2, length=2),
|
||||
]
|
||||
},
|
||||
kwargs_data={"image": ["a", "b"]},
|
||||
)
|
||||
|
||||
|
||||
def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
with pytest.raises(ValueError, match="within the token_ids sequence"):
|
||||
GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(),
|
||||
features=MultiModalFeatures(
|
||||
mm_hashes={"image": ["a"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_parser_kwargs_rejects_unknown_keys():
|
||||
"""The engine splats these into its reasoning parser's constructor, so a
|
||||
client must not be able to pass arbitrary keyword arguments."""
|
||||
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
|
||||
with pytest.raises(ValidationError, match="tokenizer"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_rust_render_sampling_params_without_nulls_validate():
|
||||
"""Rust render omits unset sampling fields; Python generate rejects nulls."""
|
||||
sampling_params = {"max_tokens": 64, "watermarking": True}
|
||||
request = GenerateRequest.model_validate(
|
||||
{"token_ids": [1, 2, 3], "sampling_params": sampling_params}
|
||||
)
|
||||
assert request.sampling_params.max_tokens == 64
|
||||
|
||||
with pytest.raises(ValidationError, match="temperature"):
|
||||
GenerateRequest.model_validate(
|
||||
{
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {**sampling_params, "temperature": None},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_output_mode_defaults_to_tokens():
|
||||
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_output_mode_text_is_accepted():
|
||||
payload = {**_base_payload(), "output_mode": "text"}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
|
||||
def test_output_mode_rejects_unsupported_levels(output_mode):
|
||||
"""Levels the server does not implement fail instead of falling back to
|
||||
tokens which would answer 200 with the wrong response shape."""
|
||||
payload = {**_base_payload(), "output_mode": output_mode}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_text_mode_rejects_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "text",
|
||||
}
|
||||
with pytest.raises(ValidationError, match="detokenize"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_tokens_mode_allows_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "tokens",
|
||||
}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_response_without_output_mode_parses_as_tokens():
|
||||
"""Existing clients and older servers never send output_mode."""
|
||||
parsed = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensResponse)
|
||||
assert parsed.output_mode == "tokens"
|
||||
|
||||
|
||||
def test_stream_chunk_without_output_mode_parses_as_tokens():
|
||||
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_and_chunk_dispatch_on_output_mode():
|
||||
response = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert isinstance(chunk, GenerateTextStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_requires_text_on_every_choice():
|
||||
with pytest.raises(ValidationError, match="text"):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0}]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
|
||||
def test_response_rejects_unknown_output_mode(output_mode):
|
||||
with pytest.raises(ValidationError):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": output_mode, "choices": []}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_accept_responses_without_output_mode():
|
||||
DerenderChatRequest.model_validate(
|
||||
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
|
||||
)
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{
|
||||
"stream": True,
|
||||
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_reject_text_responses():
|
||||
"""Text responses are already detokenized with stop strings applied, so
|
||||
re-decoding their token_ids would silently undo the truncation."""
|
||||
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatRequest.model_validate({"generate_response": text})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
|
||||
|
||||
def test_tokens_response_echoes_output_mode_without_text():
|
||||
dumped = GenerateTokensResponse(
|
||||
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
|
||||
).model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for multimodal features through the /inference/v1/generate endpoint.
|
||||
|
||||
Mirrors test_serving_tokens.py but exercises the multimodal piping
|
||||
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
|
||||
/generate -> /detokenize path. Intentionally avoids running the HF
|
||||
processor in the pytest parent process to keep os.fork() in other
|
||||
tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
RENDER_ENDPOINT = "/v1/chat/completions/render"
|
||||
DETOKENIZE_ENDPOINT = "/detokenize"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_image():
|
||||
return Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_to_generate_roundtrip(client, test_image):
|
||||
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
|
||||
|
||||
All preprocessing and detokenization happens in the server subprocess;
|
||||
the pytest parent never imports transformers or touches torch tensors.
|
||||
"""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What color is this image? Answer in one word.",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
|
||||
render_resp.raise_for_status()
|
||||
render_data = render_resp.json()
|
||||
|
||||
# Validate render output structure: keys exist and values are non-empty
|
||||
# and well-typed.
|
||||
assert "token_ids" in render_data
|
||||
assert isinstance(render_data["token_ids"], list)
|
||||
assert len(render_data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in render_data["token_ids"])
|
||||
|
||||
assert "features" in render_data
|
||||
features = render_data["features"]
|
||||
assert features is not None
|
||||
assert isinstance(features, dict)
|
||||
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) and h for h in image_hashes)
|
||||
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert isinstance(p.get("offset"), int)
|
||||
assert isinstance(p.get("length"), int)
|
||||
assert p["length"] > 0
|
||||
|
||||
assert "kwargs_data" in features
|
||||
assert "image" in features["kwargs_data"]
|
||||
assert len(features["kwargs_data"]["image"]) > 0
|
||||
|
||||
# Build generate request from render output
|
||||
generate_payload = render_data
|
||||
generate_payload["sampling_params"] = {
|
||||
"max_tokens": 10,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
|
||||
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
assert isinstance(gen_data["choices"], list)
|
||||
assert len(gen_data["choices"]) >= 1
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert isinstance(choice["token_ids"], list)
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in choice["token_ids"])
|
||||
|
||||
detok_resp = await client.post(
|
||||
DETOKENIZE_ENDPOINT,
|
||||
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
|
||||
)
|
||||
detok_resp.raise_for_status()
|
||||
detok_data = detok_resp.json()
|
||||
assert "prompt" in detok_data
|
||||
text = detok_data["prompt"]
|
||||
assert isinstance(text, str)
|
||||
assert len(text) > 0
|
||||
assert "red" in text.lower(), (
|
||||
f"Expected model to identify the red image, got: {text!r}"
|
||||
)
|
||||
|
||||
|
||||
def _collapse_placeholders(
|
||||
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
|
||||
) -> list[int]:
|
||||
"""Undo placeholder expansion: each expanded range becomes one token.
|
||||
|
||||
``/generate`` with ``content_parts`` expects unexpanded placeholders and
|
||||
expands them itself, so the roundtrip must start from a collapsed prompt.
|
||||
"""
|
||||
ranges = sorted(
|
||||
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
|
||||
)
|
||||
collapsed: list[int] = []
|
||||
pos = 0
|
||||
for offset, length in ranges:
|
||||
collapsed.extend(token_ids[pos:offset])
|
||||
collapsed.append(token_ids[offset])
|
||||
pos = offset + length
|
||||
collapsed.extend(token_ids[pos:])
|
||||
return collapsed
|
||||
|
||||
|
||||
async def _render_image_prompt(client, data_url: str) -> dict:
|
||||
render_resp = await client.post(
|
||||
RENDER_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What color is this?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
render_resp.raise_for_status()
|
||||
return render_resp.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_generates_tokens(client, test_image):
|
||||
"""Unexpanded prompt + content_parts must expand exactly like /render."""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
render_data = await _render_image_prompt(client, data_url)
|
||||
expanded = render_data["token_ids"]
|
||||
mm_placeholders = render_data["features"]["mm_placeholders"]
|
||||
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
|
||||
assert len(unexpanded) < len(expanded)
|
||||
|
||||
gen_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": unexpanded,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert gen_data["prompt_token_ids"] == expanded
|
||||
assert gen_data["mm_placeholders"] == mm_placeholders
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_streaming(client, test_image):
|
||||
"""content_parts should work with streaming."""
|
||||
import json as json_mod
|
||||
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
render_data = await _render_image_prompt(client, data_url)
|
||||
expanded = render_data["token_ids"]
|
||||
mm_placeholders = render_data["features"]["mm_placeholders"]
|
||||
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
|
||||
|
||||
async with client.stream(
|
||||
"POST",
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": unexpanded,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
|
||||
"stream": True,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json_mod.loads(line[6:]))
|
||||
|
||||
assert len(chunks) > 0
|
||||
prompt_metadata_chunks = [
|
||||
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
|
||||
]
|
||||
assert len(prompt_metadata_chunks) == 1
|
||||
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
|
||||
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders
|
||||
@@ -1,843 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
ABORT_ENDPOINT = "/inference/v1/abort_requests"
|
||||
|
||||
|
||||
def get_vocab_size(model_name):
|
||||
config = ModelConfig(
|
||||
model=model_name,
|
||||
seed=0,
|
||||
dtype="bfloat16",
|
||||
)
|
||||
return config.get_vocab_size()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def messages():
|
||||
return [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "How many countries are in the EU?"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(request):
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
|
||||
# 1 ULP when the batch dimension (M) changes, because different M
|
||||
# values cause the Tensile backend to select different tile
|
||||
# configurations with different fp32 accumulation orders. With
|
||||
# prefix caching, cache-miss prefills compute all tokens in one
|
||||
# pass (large M) while cache-hit requests compute only the
|
||||
# uncached suffix (small M), seeding a divergence that amplifies
|
||||
# through the residual stream and flips argmax tokens.
|
||||
# See: https://github.com/vllm-project/vllm/issues/33123
|
||||
#
|
||||
# Either disable prefix caching entirely, or enable it with
|
||||
# --deterministic-prefix-caching which forces cache-miss prefills
|
||||
# to split at block boundaries so the suffix GEMM shape is always
|
||||
# identical regardless of cache state.
|
||||
#
|
||||
# Option A: disable prefix caching
|
||||
"--no-enable-prefix-caching",
|
||||
#
|
||||
# Option B: deterministic prefix caching
|
||||
# "--enable-prefix-caching",
|
||||
# "--deterministic-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
extra_args = getattr(request, "param", None)
|
||||
if extra_args is not None:
|
||||
args = args + (
|
||||
list(extra_args)
|
||||
if isinstance(extra_args, (list, tuple))
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_endpoint(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
assert data["choices"][0].get("sampling_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
|
||||
"""Explicit null max_tokens must not skip the min_tokens bound."""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": None, "min_tokens": 2147483648},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "min_tokens" in resp.json()["error"]["message"]
|
||||
|
||||
followup = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 4},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=followup)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="sampling mask output is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server",
|
||||
[["--return-sampling-mask", "--logprobs-mode", "processed_logprobs"]],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_generate_sampling_mask(client):
|
||||
top_k = 5
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.8,
|
||||
"top_k": top_k,
|
||||
"top_p": 0.9,
|
||||
"ignore_eos": True,
|
||||
"seed": 0,
|
||||
"logprobs": top_k,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
choice = resp.json()["choices"][0]
|
||||
|
||||
token_ids = choice["token_ids"]
|
||||
sampling_mask = choice["sampling_mask"]
|
||||
assert sampling_mask is not None
|
||||
assert len(token_ids) == len(sampling_mask)
|
||||
|
||||
vocab_size = get_vocab_size(MODEL_NAME)
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
|
||||
assert support
|
||||
assert len(support) == len(set(support))
|
||||
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
|
||||
assert token_id in support
|
||||
# processed_logprobs: exactly the support carries finite probability.
|
||||
for top in entry["top_logprobs"]:
|
||||
in_support = top["token_id"] in support
|
||||
assert in_support == (top["logprob"] > -9999.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_defaults_max_tokens_when_omitted(client):
|
||||
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
|
||||
|
||||
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
|
||||
defaulting was wired up, every request that didn't set ``max_tokens``
|
||||
truncated mid-generation. The server should now fill it in from
|
||||
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
completion_tokens = len(data["choices"][0]["token_ids"])
|
||||
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
|
||||
# should get ~1021 tokens of output (capped at max_model_len boundary).
|
||||
assert completion_tokens > 16, (
|
||||
f"expected server-side default to exceed the legacy 16-token cap, "
|
||||
f"got {completion_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
|
||||
"""Regression: ``n > 1`` must return ``n`` choices.
|
||||
|
||||
Non-streaming requests kept ``SamplingParams``' default output kind,
|
||||
``CUMULATIVE``, so only the sequences updated during the last engine step
|
||||
reached the response and the others were silently dropped.
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_stream(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": True,
|
||||
}
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload_str = line[len("data: ") :]
|
||||
if payload_str == "[DONE]":
|
||||
break
|
||||
chunks.append(json.loads(payload_str))
|
||||
|
||||
assert len(chunks) > 0
|
||||
# Every chunk has choices with token_ids
|
||||
all_token_ids = []
|
||||
for chunk in chunks:
|
||||
assert "choices" in chunk
|
||||
assert len(chunk["choices"]) == 1
|
||||
choice = chunk["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
all_token_ids.extend(choice["token_ids"])
|
||||
|
||||
# Last chunk should have a finish_reason
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
|
||||
# Streaming should produce the same tokens as non-streaming
|
||||
non_stream_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
non_stream_data = non_stream_resp.json()
|
||||
# Just verify we got the right number of tokens
|
||||
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
|
||||
async def test_generate_logprobs(client, logprobs_value):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
"logprobs": logprobs_value,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
choice = data["choices"][0]
|
||||
assert choice["logprobs"] is not None
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
assert len(logprobs_content) == len(choice["token_ids"])
|
||||
for entry in logprobs_content:
|
||||
assert "logprob" in entry
|
||||
assert len(entry["top_logprobs"]) >= 1
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
rows = [[33], [44, 100], [55]]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": rows,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": sampling_params,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (3, 2)
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
|
||||
assert np.isneginf(scores[[0, 2], 1]).all()
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": rows}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "scored rows" in resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
for ignore_eos in [True, False]:
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
# NOTE coordinator will set this to skip detokenization
|
||||
"detokenize": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
gen_token_ids = generate_data["choices"][0]["token_ids"]
|
||||
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
if ignore_eos:
|
||||
# When ignoring EOS, only compare up to the first EOS token
|
||||
# Post-EOS generation is undefined and may differ
|
||||
eos_tokens = {
|
||||
tokenizer.eos_token_id,
|
||||
*tokenizer.extra_special_tokens_ids,
|
||||
}
|
||||
# Find first EOS in generated tokens
|
||||
eos_pos = None
|
||||
for i, tid in enumerate(gen_token_ids):
|
||||
if tid in eos_tokens:
|
||||
eos_pos = i
|
||||
break
|
||||
if eos_pos is not None:
|
||||
gen_token_ids_truncated = gen_token_ids[:eos_pos]
|
||||
generate_res = tokenizer.decode(
|
||||
gen_token_ids_truncated, skip_special_tokens=True
|
||||
)
|
||||
# Truncate completions_res to same length for comparison
|
||||
completions_res = completions_res[: len(generate_res)]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_string_workflow(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
# stop strings are only supported when detokenize is True.
|
||||
"stop": ["27 member"],
|
||||
},
|
||||
# TODO stream test is much more interesting
|
||||
"stream": False,
|
||||
}
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_resp.raise_for_status()
|
||||
|
||||
payload["sampling_params"]["stop"] = None
|
||||
generate_resp = await client.post(
|
||||
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
|
||||
)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
# NOTE This is under the responsibility of the coordinator
|
||||
# stop_checker = StopChecker(
|
||||
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
|
||||
# )
|
||||
stop_str, truncate_to = check_stop_strings(
|
||||
generate_res, len(generate_res), ["27 member"], False
|
||||
)
|
||||
assert stop_str == "27 member"
|
||||
# abort request that hit stop string (requires tokens-only mode)
|
||||
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
|
||||
# res.raise_for_status()
|
||||
generate_res = generate_res[:truncate_to]
|
||||
|
||||
# Get stop_str response from chat completions
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"stop": ["27 member"],
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_stop_strings_rejected_when_tokens_only(client):
|
||||
"""--tokens-only forces detokenize=False after request validation, so
|
||||
stop strings must be rejected explicitly rather than silently ignored."""
|
||||
sampling_params = {"max_tokens": 5, "stop": ["never"]}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": sampling_params,
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "stop strings" in resp.json()["error"]["message"]
|
||||
|
||||
sampling_params["stop"] = None
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server",
|
||||
[
|
||||
[
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
"Alice=charent/self_cognition_Alice",
|
||||
"Bob=charent/self_cognition_Bob",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
# Verify adapters are listed
|
||||
models_resp = await client.get("/v1/models")
|
||||
models_resp.raise_for_status()
|
||||
models = {m["id"] for m in models_resp.json().get("data", [])}
|
||||
assert {"Alice", "Bob"}.issubset(models)
|
||||
|
||||
# Generate using a LoRA adapter by specifying its name as the model
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
def _structured_chat_body(messages, **overrides) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 64,
|
||||
"temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"count": {"type": "integer"}},
|
||||
"required": ["count"],
|
||||
},
|
||||
},
|
||||
},
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
|
||||
render_resp = await client.post("/v1/chat/completions/render", json=body)
|
||||
render_resp.raise_for_status()
|
||||
generate_request = render_resp.json()
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
|
||||
generate_resp.raise_for_status()
|
||||
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
|
||||
# <think> tokens, so the engine gates structured outputs differently when
|
||||
# either reasoning field is dropped between render and generate.
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
|
||||
"""With thinking on, structured outputs must wait for the reasoning to
|
||||
end on generate, the same as on chat."""
|
||||
body = _structured_chat_body(messages, return_token_ids=True)
|
||||
chat_resp = await client.post("/v1/chat/completions", json=body)
|
||||
chat_resp.raise_for_status()
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is False
|
||||
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_ended(
|
||||
client, tokenizer, messages
|
||||
):
|
||||
"""`include_reasoning=false` constrains from the first token on chat, so
|
||||
generate must too, even though the engine's parser expects thinking."""
|
||||
body = _structured_chat_body(messages, include_reasoning=False)
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is True
|
||||
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
|
||||
assert isinstance(output["count"], int)
|
||||
|
||||
|
||||
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
|
||||
return tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
|
||||
def _text_mode_payload(token_ids, **sampling_params) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
|
||||
"output_mode": "text",
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == "text"
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
chat_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
assert data["choices"][0]["text"] == chat_content
|
||||
assert data["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
payload = _text_mode_payload(token_ids)
|
||||
|
||||
non_stream = await client.post(GEN_ENDPOINT, json=payload)
|
||||
non_stream.raise_for_status()
|
||||
|
||||
chunks = []
|
||||
async with client.stream(
|
||||
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json.loads(line[len("data: ") :]))
|
||||
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
|
||||
assert streamed_text == non_stream.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
pytest.param(
|
||||
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
|
||||
),
|
||||
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
|
||||
],
|
||||
)
|
||||
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
|
||||
payload = {
|
||||
**_text_mode_payload([1, 2, 3], max_tokens=5),
|
||||
**overrides,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
|
||||
assert resp.status_code == 400
|
||||
assert expected in resp.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_text_mode_rejected_when_tokens_only(client):
|
||||
"""Without a tokenizer the detokenizer returns "", so the request must
|
||||
fail instead of answering 200 with empty text."""
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
|
||||
assert resp.status_code == 400
|
||||
assert "requires a tokenizer" in resp.json()["error"]["message"]
|
||||
|
||||
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
|
||||
payload["sampling_params"]["detokenize"] = False
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
|
||||
async def test_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages, output_mode
|
||||
):
|
||||
"""The final output after an abort has no new token IDs, but the
|
||||
stream must still deliver its finish reason."""
|
||||
payload = {
|
||||
**_text_mode_payload(
|
||||
_chat_prompt_token_ids(tokenizer, messages),
|
||||
max_tokens=900,
|
||||
ignore_eos=True,
|
||||
),
|
||||
"output_mode": output_mode,
|
||||
"stream": True,
|
||||
"request_id": f"{output_mode}-abort-e2e",
|
||||
}
|
||||
|
||||
chunks = []
|
||||
aborted = False
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
chunks.append(chunk)
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
|
||||
abort = await client.post(
|
||||
ABORT_ENDPOINT,
|
||||
json={"request_ids": [chunk["request_id"]]},
|
||||
)
|
||||
assert abort.status_code == 200
|
||||
aborted = True
|
||||
|
||||
assert aborted
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
|
||||
final_choice = chunks[-1]["choices"][0]
|
||||
assert final_choice["finish_reason"] == "abort"
|
||||
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
|
||||
assert generated < 900
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_abort_requests_is_served_without_tokens_only(client):
|
||||
resp = await client.post(
|
||||
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = await client.post(ABORT_ENDPOINT, json={})
|
||||
assert resp.status_code == 400
|
||||
|
||||
# The unauthenticated path is only served with --tokens-only.
|
||||
resp = await client.post(
|
||||
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own integer token id."""
|
||||
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
|
||||
)
|
||||
token_ids = {e.token_id for e in result.content[0].top_logprobs}
|
||||
assert token_ids == {262, 257}, f"got {token_ids}"
|
||||
|
||||
|
||||
def test_sampled_entry_carries_token_id_and_rank():
|
||||
"""The sampled token is identified by id, and the engine's rank survives."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[
|
||||
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
|
||||
],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 262
|
||||
assert entry.logprob == -0.1
|
||||
assert entry.rank == 1
|
||||
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
|
||||
# No tokenizer on the generate server: no string token, no bytes.
|
||||
assert not hasattr(entry, "token")
|
||||
assert not hasattr(entry, "bytes")
|
||||
|
||||
|
||||
def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[5],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 5
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs == []
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
|
||||
|
||||
|
||||
def test_logprobs_minus_one_emits_all_tokens():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
|
||||
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
|
||||
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token == "token_id:7"
|
||||
assert entry.bytes is None
|
||||
assert all(t.bytes is None for t in entry.top_logprobs)
|
||||
|
||||
|
||||
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
tokenizer = MagicMock()
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
tokenizer.decode.assert_not_called()
|
||||
|
||||
|
||||
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
assert result.content[0].top_logprobs[0].token == "<7>"
|
||||
assert result.content[1].bytes == list(b"<9>")
|
||||
|
||||
|
||||
def test_sampled_token_outside_topk_comes_first():
|
||||
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
|
||||
all k + 1 candidates when the sampled token is outside the top k; derender
|
||||
applies each endpoint's cut (#59513)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[50],
|
||||
top_logprobs=[
|
||||
{
|
||||
50: Logprob(-3.0, rank=5),
|
||||
10: Logprob(-0.2, rank=1),
|
||||
20: Logprob(-1.0, rank=2),
|
||||
}
|
||||
],
|
||||
)
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
|
||||
|
||||
|
||||
def test_logprob_is_required_on_the_wire():
|
||||
"""A payload missing logprob is rejected rather than read as -9999."""
|
||||
with pytest.raises(ValidationError):
|
||||
GenerateLogProb.model_validate({"token_id": 1})
|
||||
|
||||
|
||||
def test_rank_zero_from_a_nan_logprob_is_none():
|
||||
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs[0].rank is None
|
||||
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
|
||||
|
||||
def test_text_logprobs_clamp_a_nan_logprob():
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
|
||||
tokenizer=MagicMock(),
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
@@ -120,7 +120,7 @@ class TestAffectedModulesUseSanitize:
|
||||
@pytest.mark.parametrize(
|
||||
"module",
|
||||
[
|
||||
"vllm.entrypoints.speech_to_text.realtime.connection",
|
||||
"vllm.entrypoints.serve.exception_handling.handlers.http",
|
||||
],
|
||||
)
|
||||
def test_module_calls_sanitize_message(self, module: str):
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import shutil
|
||||
from contextlib import suppress
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
BADREQUEST_CASES = [
|
||||
(
|
||||
"test_rank",
|
||||
{"r": 1024},
|
||||
"is greater than max_lora_rank",
|
||||
),
|
||||
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
|
||||
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
|
||||
(
|
||||
"test_modules_to_save",
|
||||
{"modules_to_save": ["lm_head"]},
|
||||
"Unsupported modules_to_save",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[True])
|
||||
def server_with_lora_modules_json(request, qwen3_lora_files):
|
||||
# Define the json format LoRA module configurations
|
||||
lora_module_1 = {
|
||||
"name": "qwen3-lora",
|
||||
"path": qwen3_lora_files,
|
||||
"base_model_name": MODEL_NAME,
|
||||
}
|
||||
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
json.dumps(lora_module_1),
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"64",
|
||||
]
|
||||
|
||||
# Enable the /v1/load_lora_adapter endpoint
|
||||
envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server_with_lora_modules_json):
|
||||
async with server_with_lora_modules_json.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
served_model = models[0]
|
||||
lora_models = models[1:]
|
||||
assert served_model.id == MODEL_NAME
|
||||
assert served_model.root == MODEL_NAME
|
||||
assert served_model.parent is None
|
||||
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
|
||||
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
|
||||
assert lora_models[0].id == "qwen3-lora"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
|
||||
)
|
||||
# Ensure adapter loads before querying /models
|
||||
assert "success" in response
|
||||
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
dynamic_lora_model = models[-1]
|
||||
assert dynamic_lora_model.root == qwen3_lora_files
|
||||
assert dynamic_lora_model.parent == MODEL_NAME
|
||||
assert dynamic_lora_model.id == "qwen3-lora-3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_with_same_name_replaces_inplace(
|
||||
client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
|
||||
):
|
||||
"""Test that loading a LoRA adapter with the same name replaces it inplace."""
|
||||
adapter_name = "replaceable-adapter"
|
||||
messages = [
|
||||
{"content": "Follow the instructions to make animal noises", "role": "system"},
|
||||
{"content": "Make your favorite animal noise.", "role": "user"},
|
||||
]
|
||||
|
||||
# Load LoRA that makes model meow
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
completion = await client.chat.completions.create(
|
||||
model=adapter_name,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
assert "Meow Meow Meow" in completion.choices[0].message.content
|
||||
|
||||
# Load LoRA that makes model woof
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={
|
||||
"lora_name": adapter_name,
|
||||
"lora_path": qwen3_woofing_lora_files,
|
||||
"load_inplace": True,
|
||||
},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
completion = await client.chat.completions.create(
|
||||
model=adapter_name,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
assert "Woof Woof Woof" in completion.choices[0].message.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_with_load_inplace_false_errors(
|
||||
client: openai.AsyncOpenAI, qwen3_meowing_lora_files
|
||||
):
|
||||
"""Test that load_inplace=False returns an error when adapter already exists."""
|
||||
adapter_name = "test-load-inplace-false"
|
||||
|
||||
# Load LoRA adapter first time (should succeed)
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
# Try to load the same adapter again with load_inplace=False (should fail)
|
||||
with pytest.raises(openai.BadRequestError) as exc_info:
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={
|
||||
"lora_name": adapter_name,
|
||||
"lora_path": qwen3_meowing_lora_files,
|
||||
},
|
||||
)
|
||||
|
||||
# Verify the error message
|
||||
assert "already been loaded" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
|
||||
invalid_files = tmp_path / "invalid_files"
|
||||
invalid_files.mkdir()
|
||||
(invalid_files / "adapter_config.json").write_text("this is not json")
|
||||
|
||||
with pytest.raises(openai.InternalServerError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "invalid-json", "lora_path": str(invalid_files)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES)
|
||||
async def test_dynamic_lora_badrequests(
|
||||
client: openai.AsyncOpenAI,
|
||||
tmp_path,
|
||||
qwen3_lora_files,
|
||||
test_name: str,
|
||||
config_change: dict,
|
||||
expected_error: str,
|
||||
):
|
||||
# Create test directory
|
||||
test_dir = tmp_path / test_name
|
||||
|
||||
# Copy adapter files
|
||||
shutil.copytree(qwen3_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
with open(config_path) as f:
|
||||
adapter_config = json.load(f)
|
||||
# Apply configuration changes
|
||||
adapter_config.update(config_change)
|
||||
|
||||
# Save modified configuration
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(adapter_config, f)
|
||||
|
||||
# Test loading the adapter
|
||||
with pytest.raises(openai.InternalServerError, match=expected_error):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": test_name, "lora_path": str(test_dir)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_lora_adapters(
|
||||
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
||||
):
|
||||
"""Validate that many loras can be dynamically registered and inferenced
|
||||
with concurrently"""
|
||||
|
||||
# This test file configures the server with --max-cpu-loras=2 and this test
|
||||
# will concurrently load 10 adapters, so it should flex the LRU cache
|
||||
async def load_and_run_adapter(adapter_name: str):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
|
||||
)
|
||||
for _ in range(3):
|
||||
await client.completions.create(
|
||||
model=adapter_name,
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
|
||||
lora_tasks = []
|
||||
for i in range(10):
|
||||
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
|
||||
|
||||
results, _ = await asyncio.wait(lora_tasks)
|
||||
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loading_invalid_adapters_does_not_break_others(
|
||||
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
||||
):
|
||||
invalid_files = tmp_path / "invalid_files"
|
||||
invalid_files.mkdir()
|
||||
(invalid_files / "adapter_config.json").write_text("this is not json")
|
||||
|
||||
stop_good_requests_event = asyncio.Event()
|
||||
|
||||
async def run_good_requests(client):
|
||||
# Run chat completions requests until event set
|
||||
|
||||
results = []
|
||||
|
||||
while not stop_good_requests_event.is_set():
|
||||
try:
|
||||
batch = await client.completions.create(
|
||||
model="qwen3-lora",
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
results.append(batch)
|
||||
except Exception as e:
|
||||
results.append(e)
|
||||
|
||||
return results
|
||||
|
||||
# Create task to run good requests
|
||||
good_task = asyncio.create_task(run_good_requests(client))
|
||||
|
||||
# Run a bunch of bad adapter loads
|
||||
for _ in range(25):
|
||||
with suppress(openai.NotFoundError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
|
||||
)
|
||||
for _ in range(25):
|
||||
with suppress(openai.InternalServerError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "invalid", "lora_path": str(invalid_files)},
|
||||
)
|
||||
|
||||
# Ensure all the running requests with lora adapters succeeded
|
||||
stop_good_requests_event.set()
|
||||
results = await good_task
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
|
||||
# Ensure we can load another adapter and run it
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "valid", "lora_path": qwen3_lora_files},
|
||||
)
|
||||
await client.completions.create(
|
||||
model="valid",
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_beam_search_with_lora_adapters(
|
||||
client: openai.AsyncOpenAI,
|
||||
tmp_path,
|
||||
qwen3_lora_files,
|
||||
):
|
||||
"""Validate that async beam search can be used with lora."""
|
||||
|
||||
async def load_and_run_adapter(adapter_name: str):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
|
||||
)
|
||||
for _ in range(3):
|
||||
await client.completions.create(
|
||||
model=adapter_name,
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
|
||||
lora_tasks = []
|
||||
for i in range(3):
|
||||
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
|
||||
|
||||
results, _ = await asyncio.wait(lora_tasks)
|
||||
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
@@ -214,7 +214,6 @@ def test_request_logger_log_outputs_integration():
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_embeds=None,
|
||||
params=None,
|
||||
lora_request=None,
|
||||
)
|
||||
|
||||
request_logger.log_outputs(
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
|
||||
def add_attention_backend(server_args, attention_config):
|
||||
"""Append attention backend CLI arg if specified.
|
||||
|
||||
Args:
|
||||
server_args: List of server arguments to extend in-place.
|
||||
attention_config: Dict with 'backend' key, or None.
|
||||
|
||||
"""
|
||||
if attention_config and "backend" in attention_config:
|
||||
server_args.extend(["--attention-backend", attention_config["backend"]])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mary_had_lamb():
|
||||
path = AudioAsset("mary_had_lamb").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def winning_call():
|
||||
path = AudioAsset("winning_call").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def foscolo():
|
||||
# Test translation it->en
|
||||
path = AudioAsset("azacinto_foscolo").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def transcription_server_with_force_include_usage():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--enable-force-include-usage",
|
||||
"--gpu-memory-utilization",
|
||||
"0.2",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transcription_client_with_force_include_usage(
|
||||
transcription_server_with_force_include_usage,
|
||||
):
|
||||
async with (
|
||||
transcription_server_with_force_include_usage.get_async_client() as async_client
|
||||
):
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_with_enable_force_include_usage(
|
||||
transcription_client_with_force_include_usage, winning_call
|
||||
):
|
||||
res = (
|
||||
await transcription_client_with_force_include_usage.audio.transcriptions.create(
|
||||
model="openai/whisper-large-v3-turbo",
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
)
|
||||
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
usage = chunk.usage
|
||||
assert isinstance(usage, dict)
|
||||
assert usage["prompt_tokens"] > 0
|
||||
assert usage["completion_tokens"] > 0
|
||||
assert usage["total_tokens"] > 0
|
||||
else:
|
||||
assert not hasattr(chunk, "usage")
|
||||
Reference in New Issue
Block a user