[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
-19
View File
@@ -1534,29 +1534,10 @@ def num_gpus_available():
temp_dir = tempfile.gettempdir()
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
@pytest.fixture
def dummy_opt_path():
json_path = os.path.join(_dummy_opt_path, "config.json")
if not os.path.exists(_dummy_opt_path):
hf_api().snapshot_download(
repo_id="facebook/opt-125m",
local_dir=_dummy_opt_path,
ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
)
assert os.path.exists(json_path)
with open(json_path) as f:
config = json.load(f)
config["architectures"] = ["MyOPTForCausalLM"]
with open(json_path, "w") as f:
json.dump(config, f)
return _dummy_opt_path
@pytest.fixture
def dummy_llava_path():
json_path = os.path.join(_dummy_llava_path, "config.json")
@@ -39,7 +39,6 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
sampling_params=params,
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
-28
View File
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.plugins_tests.test_oot_registration_online import (
run_and_test_dummy_opt_api_server,
)
def test_distributed_oot(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
@pytest.mark.parametrize("enforce_eager", [True, False])
def test_distributed_oot_pp_raw_input_tokens(dummy_opt_path: str, enforce_eager: bool):
# The model asserts that input_ids reaches every pipeline stage.
overrides = {"architectures": ["MyOPTRawInputTokensForCausalLM"]}
extra_args = ["-pp", "2", "--hf-overrides", json.dumps(overrides)]
if enforce_eager:
extra_args.append("--enforce-eager")
run_and_test_dummy_opt_api_server(
dummy_opt_path,
extra_args=extra_args,
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
)
-10
View File
@@ -24,9 +24,7 @@ class _LegacyBlockStored(
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
@@ -58,9 +56,7 @@ class _PreSessionBlockStored(
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
@@ -79,9 +75,7 @@ def _make_block_stored(
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=group_idx,
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
locality=locality,
@@ -183,9 +177,7 @@ def test_block_stored_locality_is_wire_compatible():
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
)
@@ -221,9 +213,7 @@ def test_block_stored_session_id_is_wire_compatible():
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
locality="LOCAL",
-34
View File
@@ -65,37 +65,3 @@ def sample_structured_outputs_choices():
"Swift",
"Kotlin",
]
@pytest.fixture(scope="session")
def qwen3_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
@pytest.fixture(scope="session")
def qwen3_woofing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def opt125_lora_files() -> str:
"""Download opt-125m LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(
repo_id="peft-internal-testing/opt-125m-dummy-lora"
)
+4 -124
View File
@@ -1,28 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.utils import VLLM_PATH
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.exceptions import VLLMValidationError
from vllm.utils.argparse_utils import FlexibleArgumentParser
LORA_MODULE = {
"name": "module2",
"path": "/path/to/module2",
"base_model_name": "llama",
}
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
assert CHATML_JINJA_PATH.exists()
def _build_vllm_parsers():
vllm_parser = FlexibleArgumentParser()
@@ -96,87 +84,6 @@ def test_logging_config_cli_args(serve_parser):
assert config.pylogging_config_file == "/tmp/flat.json"
### Tests for LoRA module parsing
def test_valid_key_value_format(serve_parser):
# Test old format: name=path
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
]
)
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
assert args.lora_modules == expected
def test_valid_json_format(serve_parser):
# Test valid JSON format input
args = serve_parser.parse_args(
[
"--lora-modules",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
]
assert args.lora_modules == expected
def test_invalid_json_format(serve_parser):
# Test invalid JSON format input, missing closing brace
with pytest.raises(SystemExit):
serve_parser.parse_args(
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
)
def test_invalid_type_error(serve_parser):
# Test type error when values are not JSON or key=value
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
"invalid_format", # This is not JSON or key=value format
]
)
def test_invalid_json_field(serve_parser):
# Test valid JSON format but missing required fields
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
'{"name": "module4"}', # Missing required 'path' field
]
)
def test_empty_values(serve_parser):
# Test when no LoRA modules are provided
args = serve_parser.parse_args(["--lora-modules", ""])
assert args.lora_modules == []
def test_multiple_valid_inputs(serve_parser):
# Test multiple valid inputs (both old and JSON format)
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module1", path="/path/to/module1"),
LoRAModulePath(
name="module2", path="/path/to/module2", base_model_name="llama"
),
]
assert args.lora_modules == expected
### Tests for serve argument validation that run prior to loading
@@ -241,10 +148,12 @@ def test_reasoning_parser_none_disables_parser(serve_parser, value, expected):
assert args.reasoning_parser == expected
def test_chat_template_validation_for_happy_paths(serve_parser):
def test_chat_template_validation_for_happy_paths(serve_parser, tmp_path):
"""Ensure validation passes if the chat template exists."""
chat_template = tmp_path / "template.jinja"
chat_template.write_text("{{ messages }}")
args = serve_parser.parse_args(
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
args=["--chat-template", chat_template.absolute().as_posix()]
)
validate_parsed_serve_args(args)
@@ -389,32 +298,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
else:
with pytest.raises(raises):
vllm_parser.parse_args(args=args)
### Tests for LoRA target modules parsing
def test_lora_target_modules_single(serve_parser):
"""Test parsing single lora-target-modules argument."""
args = serve_parser.parse_args(
args=["--enable-lora", "--lora-target-modules", "o_proj"]
)
assert args.lora_target_modules == ["o_proj"]
def test_lora_target_modules_multiple(serve_parser):
"""Test parsing multiple lora-target-modules arguments."""
args = serve_parser.parse_args(
args=[
"--enable-lora",
"--lora-target-modules",
"o_proj",
"qkv_proj",
"down_proj",
]
)
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
def test_lora_target_modules_default_none(serve_parser):
"""Test that lora-target-modules defaults to None."""
args = serve_parser.parse_args(args=[])
assert args.lora_target_modules is None
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.conftest import AudioTestAssets
from tests.utils import RemoteOpenAIServer
from vllm.transformers_utils.repo_utils import hf_api
# NOTE - the tests in this module are currently analogous to test_chat, but are
# separated to avoid OOM killing due to module-scoped servers, since we
# need a multimodal model for these tests.
# Contains a modality specific lora alongside the base model
MULTIMODAL_MODEL_NAME = hf_api().snapshot_download(
"microsoft/Phi-4-multimodal-instruct"
)
AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
@pytest.fixture(scope="module")
def multimodal_server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"half",
"--max-model-len",
"4096",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"speech={AUDIO_LORA_PATH}",
"--max-lora-rank",
"320",
"--max-num-seqs",
"2",
"--trust-remote-code",
"--gpu-memory-utilization",
"0.8",
"--default-mm-loras",
f'{{"audio": "{AUDIO_LORA_PATH}"}}',
]
with RemoteOpenAIServer(
MULTIMODAL_MODEL_NAME, args, max_wait_seconds=480
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def multi_modal_client(multimodal_server):
async with multimodal_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
# base model with default lora should give the same response as lora model
"model_name",
[MULTIMODAL_MODEL_NAME, "speech"],
)
async def test_default_mm_lora_chat_completions(
model_name: str,
multi_modal_client: openai.AsyncOpenAI,
audio_assets: AudioTestAssets,
):
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Can you transcribe this audio?",
},
{
"type": "audio_url",
"audio_url": {"url": audio_assets[0].url},
},
],
}
]
chat_completion = await multi_modal_client.chat.completions.create(
model=model_name, messages=messages, max_completion_tokens=128, temperature=0.0
)
assert len(chat_completion.choices) > 0
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
assert message.content == ACTIVE_MM_LORA_RESPONSE
@@ -27,15 +27,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@pytest.fixture(scope="module")
def zephyr_lora_files():
"""Download zephyr LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
@pytest.fixture(scope="module")
def server(zephyr_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -43,14 +35,6 @@ def server(zephyr_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"zephyr-lora={zephyr_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -67,9 +51,8 @@ async def client(server):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# first test base model, then test loras
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -91,9 +74,8 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -119,7 +101,7 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -145,7 +127,7 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -254,7 +236,7 @@ async def test_more_than_one_prompt_logprobs_chat(
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -297,9 +279,8 @@ async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -345,7 +326,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
["HuggingFaceH4/zephyr-7b-beta"],
)
async def test_chat_completion_stream_options(
client: openai.AsyncOpenAI, model_name: str
@@ -9,7 +9,6 @@ import openai # use the official client for correctness check
import pytest
import pytest_asyncio
# downloading lora to test lora requests
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
# any model with a chat template should work here
+2 -15
View File
@@ -9,12 +9,10 @@ from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
# generation quality here
@pytest.fixture(scope="module")
def server(qwen3_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -22,14 +20,6 @@ def server(qwen3_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"qwen3-lora={qwen3_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -45,12 +35,9 @@ async def client(server):
@pytest.mark.asyncio
async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
async def test_check_models(client: openai.AsyncOpenAI):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token-offsets request/response protocol wiring:
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
serialization boundary. Plain Pydantic field storage is not retested here.
"""
from unittest.mock import Mock
from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams
def _model_config() -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = 128
return model_config
def test_completion_flag_forwarded_to_tok_params():
"""build_tok_params must forward return_token_offsets, defaulting to
False (zero behavioral change for existing callers) and coercing JSON
null to False via the bool() guard."""
cfg = _model_config()
default = CompletionRequest(model="m", prompt="hi")
assert default.build_tok_params(cfg).return_token_offsets is False
on = CompletionRequest(model="m", prompt="hi", return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = CompletionRequest(model="m", prompt="hi", return_token_offsets=None)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_chat_flag_forwarded_to_tok_params():
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
its return_token_offsets forwarding is verified independently."""
cfg = _model_config()
messages = [{"role": "user", "content": "hi"}]
default = ChatCompletionRequest(model="m", messages=messages)
assert default.build_tok_params(cfg).return_token_offsets is False
on = ChatCompletionRequest(model="m", messages=messages, return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = ChatCompletionRequest(
model="m", messages=messages, return_token_offsets=None
)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_generate_request_token_offsets_default_none():
"""Defaults to None so existing /v1/.../render responses are unchanged."""
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=SamplingParams())
assert req.token_offsets is None
def test_generate_request_token_offsets_survive_json_round_trip():
"""GenerateRequest crosses the disagg serialization boundary; the
tuple[int, int] offsets must survive model_dump and re-validate."""
req = GenerateRequest(
token_ids=[10, 20],
sampling_params=SamplingParams(),
token_offsets=[(0, 1), (1, 3)],
)
dumped = req.model_dump()
assert dumped["token_offsets"] == [(0, 1), (1, 3)]
# Re-validate from the dumped dict (sampling_params doesn't round-trip
# cleanly via dump, so re-inject a fresh instance).
again = GenerateRequest.model_validate(
{**dumped, "sampling_params": SamplingParams()}
)
assert again.token_offsets == [(0, 1), (1, 3)]
@@ -1,125 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from argparse import Namespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import (
GUARDED_PREFIX,
AuthenticationMiddleware,
)
from vllm.tasks import SupportedTask
GENERATE_PATH = "/inference/v1/generate"
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...],
*,
tokens_only: bool = False,
enable_scale_out: bool = False,
) -> set[str]:
app = FastAPI()
args = Namespace(
tokens_only=tokens_only,
enable_scale_out=enable_scale_out,
enable_fault_tolerance=False,
)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
@pytest.mark.parametrize("authorized", [False, True])
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
app = FastAPI()
register_scale_out_api_routers(app, ("generate",))
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
headers = {"Authorization": "Bearer secret"} if authorized else {}
client = TestClient(app)
response = client.post(path, content="not JSON", headers=headers)
assert response.status_code == (403 if authorized else 401)
if authorized:
assert "--enable-scale-out" in response.json()["error"]["message"]
assert response.json()["error"]["type"] == "PermissionDeniedError"
assert path not in app.openapi()["paths"]
assert client.post("/unknown", headers=headers).status_code == 404
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert [
record.levelno
for record in caplog.records
if "--enable-scale-out" in record.message
] == [logging.INFO]
@pytest.mark.parametrize("tokens_only", [False, True])
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
app = FastAPI()
app.state.args = Namespace(
tokens_only=tokens_only, enable_scale_out=not tokens_only
)
register_scale_out_api_routers(app, ("generate",))
response = TestClient(app).post(
GENERATE_PATH,
content="not JSON",
headers={"Content-Type": "application/json"},
)
assert response.status_code == 422
assert response.json()["detail"][0]["type"] == "json_invalid"
def test_scale_out_routes_can_be_enabled_for_generate_server():
paths = registered_paths(("generate",), enable_scale_out=True)
assert paths >= SCALE_OUT_PATHS
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
def test_abort_route_without_tokens_only_requires_api_key():
"""A decode pool that loads a tokenizer serves generate without
--tokens-only and still needs to abort, but only behind --api-key."""
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
assert ABORT_PATH in paths
assert ABORT_PATH.startswith(GUARDED_PREFIX)
assert UNAUTHENTICATED_ABORT_PATH not in paths
def test_unauthenticated_abort_route_is_not_registered_twice():
"""A pre-existing /abort_requests route must not be shadowed by a second
registration from the scale-out router."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@app.post(UNAUTHENTICATED_ABORT_PATH)
async def existing_abort_requests():
return None
attach_router(app)
paths = [route.path for route in app.routes]
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
assert ABORT_PATH in paths
@@ -1,419 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Roundtrip tests for multimodal serde used by the
token_in_token_out generate endpoint.
"""
import pytest
import torch
from pydantic import ValidationError
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
merge_mm_kwargs_items,
mm_kwargs_from_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.inputs import mm_input
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalFlatField,
MultiModalKwargsItem,
MultiModalSharedField,
PlaceholderRange,
)
def test_mm_kwargs_item_roundtrip():
"""Full roundtrip test with all three field types and multiple dtypes."""
e1 = MultiModalFieldElem(
data=torch.zeros(1000, dtype=torch.bfloat16),
field=MultiModalBatchedField(),
)
e2 = MultiModalFieldElem(
data=torch.ones(100, dtype=torch.int32),
field=MultiModalSharedField(batch_size=4),
)
e3 = MultiModalFieldElem(
data=torch.randn(20, dtype=torch.float32),
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
)
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
encoded = encode_mm_kwargs_item(item)
# Encoded result is a base64 string
assert isinstance(encoded, str)
decoded = decode_mm_kwargs_item(encoded)
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
def test_mm_kwargs_item_none_data():
"""Roundtrip with None data field."""
elem = MultiModalFieldElem(
data=None,
field=MultiModalSharedField(batch_size=2),
)
item = MultiModalKwargsItem({"empty": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
assert decoded["empty"].data is None
assert isinstance(decoded["empty"].field, MultiModalSharedField)
def test_mm_kwargs_item_nested_tensors():
"""Roundtrip with nested tensor data."""
nested = [torch.randn(3, 4), torch.randn(5, 4)]
elem = MultiModalFieldElem(
data=nested,
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"nested": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
decoded_data = decoded["nested"].data
assert len(decoded_data) == 2
assert torch.equal(nested[0], decoded_data[0])
assert torch.equal(nested[1], decoded_data[1])
def test_mm_features_with_kwargs_data():
"""Test that MultiModalFeatures can carry serialized tensor data."""
elem = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"pixel_values": elem})
encoded = encode_mm_kwargs_item(item)
features = MultiModalFeatures(
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
kwargs_data={"image": [encoded]},
)
# JSON roundtrip
json_str = features.model_dump_json()
features2 = MultiModalFeatures.model_validate_json(json_str)
assert features2.mm_hashes == {"image": ["abc123"]}
assert features2.kwargs_data is not None
assert len(features2.kwargs_data["image"]) == 1
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
assert torch.equal(elem.data, decoded["pixel_values"].data)
def _image_engine_input(
*,
pixel_keep_on_cpu: bool = False,
grid_keep_on_cpu: bool = True,
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3],
mm_kwargs={"image": [item]},
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
)
return engine_input, pixel_values, image_grid_thw
def test_render_extracts_metadata_fields_separately():
"""Render metadata contains CPU fields but excludes encoder inputs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
features = extract_mm_features(engine_input)
assert features is not None
assert features.mm_metadata is not None
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
assert features.kwargs_data is not None
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
assert set(full_data) == {"pixel_values", "image_grid_thw"}
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
def test_extract_without_kwargs_keeps_layout():
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
full = extract_mm_features(engine_input)
layout = extract_mm_features(engine_input, mm_payload="none")
assert full is not None and layout is not None
assert layout.kwargs_data is None
assert layout.mm_metadata is None
assert layout.mm_hashes == full.mm_hashes
assert layout.mm_placeholders == full.mm_placeholders
_MESSAGES = [{"role": "user", "content": "hi"}]
@pytest.mark.parametrize(
("request_cls", "body"),
[
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
(CompletionRequest, {"model": "m", "prompt": "hi"}),
(ResponsesRequest, {"model": "m", "input": "hi"}),
],
)
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
"""Clients of the deprecated bool keep getting layout-only responses."""
assert request_cls.model_validate(body).mm_payload == "tensors"
for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
request = request_cls.model_validate(
{**body, "return_mm_kwargs": return_mm_kwargs}
)
assert request.mm_payload == expected
explicit = request_cls.model_validate(
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
)
assert explicit.mm_payload == "tensors"
with pytest.raises(ValidationError):
request_cls.model_validate({**body, "mm_payload": "source"})
def test_extract_includes_declared_placeholder_metadata_fields():
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
grid_keep_on_cpu=False
)
features = extract_mm_features(
engine_input,
metadata_fields_for=lambda modality: (
{"image_grid_thw"} if modality == "image" else set()
),
)
assert features is not None
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_legacy_kwargs_only_generate_keeps_full_payload():
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
assert rendered.kwargs_data is not None
features = MultiModalFeatures(
mm_hashes=rendered.mm_hashes,
mm_placeholders=rendered.mm_placeholders,
kwargs_data=rendered.kwargs_data,
)
merged = mm_kwargs_from_features(features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_metadata_only_generate_requires_ec_transfer_params():
engine_input, _, _ = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": {
"mm_hashes": rendered.mm_hashes,
"mm_placeholders": {
modality: [p.model_dump() for p in ranges]
for modality, ranges in rendered.mm_placeholders.items()
},
"mm_metadata": rendered.mm_metadata,
},
}
try:
GenerateRequest.model_validate(payload)
raise AssertionError("expected metadata-only request without EC to fail")
except ValidationError as exc:
assert "ec_transfer_params" in str(exc)
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
request = GenerateRequest.model_validate(payload)
assert request.features is not None
assert request.features.kwargs_data is None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"image_grid_thw"}
@pytest.mark.parametrize("modality", ["image", "video"])
@pytest.mark.parametrize(
("has_metadata", "has_ec"),
[(True, False), (True, True), (False, False)],
ids=["metadata-without-ec", "metadata-with-ec", "cache-hit-without-ec"],
)
def test_mixed_items_require_ec_only_for_metadata_only(
modality: str, has_metadata: bool, has_ec: bool
):
"""A full item must not bypass the EC requirement for another item."""
engine_input, _, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
features = rendered.model_dump()
metadata = (
encode_mm_kwargs_item(
MultiModalKwargsItem({f"{modality}_grid_thw": image_grid_thw})
)
if has_metadata
else None
)
second_item = {
"mm_hashes": "second-item",
"mm_placeholders": {"offset": 2, "length": 1},
"kwargs_data": None,
"mm_metadata": metadata,
}
for key, value in second_item.items():
features[key].setdefault(modality, []).append(value)
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": features,
}
if has_ec:
payload["ec_transfer_params"] = {"second-item": {"peer_host": "10.0.0.1"}}
if has_metadata and not has_ec:
with pytest.raises(ValidationError, match="ec_transfer_params"):
GenerateRequest.model_validate(payload)
else:
GenerateRequest.model_validate(payload)
def test_kwargs_and_metadata_generate_does_not_require_ec():
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
request = GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": rendered.model_dump(),
}
)
assert request.features is not None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_cache_hit_null_kwargs_keeps_metadata_slot():
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3, 4],
mm_kwargs={"image": [item, None]},
mm_hashes={"image": ["hit", "cached"]},
mm_placeholders={
"image": [
PlaceholderRange(offset=0, length=1),
PlaceholderRange(offset=2, length=1),
]
},
)
features = extract_mm_features(engine_input)
assert features is not None
assert features.kwargs_data is not None
assert features.mm_metadata is not None
assert features.kwargs_data["image"][1] is None
assert features.mm_metadata["image"][1] is None
assert len(features.mm_metadata["image"]) == 2
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_features_alignment_rejects_length_mismatch():
try:
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
)
except ValidationError:
return
raise AssertionError("expected placeholder length mismatch to fail")
def test_metadata_can_replace_or_extend_full_mm_data():
"""Metadata-only requests retain the fields required by the model."""
metadata_elem = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
pixel_elem = MultiModalFieldElem(
data=torch.randn(5, 3),
field=MultiModalBatchedField(),
)
full_item = MultiModalKwargsItem(
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
)
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
merged = merge_mm_kwargs_items(full_item, metadata_item)
assert merged is not None
assert set(merged) == {"pixel_values", "image_grid_thw"}
@@ -1,266 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token_in_token_out request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and
fail loudly if the validator semantics ever drift.
"""
import json
import pytest
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
GenerateStreamResponse,
GenerateTextResponse,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.sampling_params import SamplingParams
def _base_payload() -> dict:
return {"token_ids": [1, 2, 3], "sampling_params": {}}
def test_omitted_max_tokens_is_not_provided():
"""Body without ``max_tokens`` must surface as 'not provided' so the
server can apply its own default instead of the dataclass 16."""
req = GenerateRequest.model_validate(_base_payload())
# SamplingParams' dataclass default leaks through the parsed instance —
# this is exactly the bug the server-side defaulting works around.
assert req.sampling_params.max_tokens == 16
assert not req.is_sampling_param_provided("max_tokens")
def test_explicit_max_tokens_is_provided():
"""Even when the client picks the same value as the dataclass default,
it must register as explicitly set so the server won't override it."""
payload = _base_payload()
payload["sampling_params"] = {"max_tokens": 16}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 16
assert req.is_sampling_param_provided("max_tokens")
payload["sampling_params"] = {"max_tokens": 256}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 256
assert req.is_sampling_param_provided("max_tokens")
def test_other_fields_tracked_independently():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.0}
req = GenerateRequest.model_validate(payload)
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_json_roundtrip_preserves_provided_keys():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.5}
req = GenerateRequest.model_validate_json(json.dumps(payload))
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_internal_instance_construction_treats_all_as_provided():
"""When internal callers build ``GenerateRequest`` from a pre-resolved
``SamplingParams`` instance, every field is considered explicitly set
so server-side defaulting can't clobber values resolved upstream."""
sp = SamplingParams(max_tokens=500, temperature=0.0)
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
assert req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
# And keys we never touched should also count as provided in this path.
assert req.is_sampling_param_provided("top_p")
def test_multimodal_features_reject_mismatched_parallel_fields():
with pytest.raises(ValueError, match="same length"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
kwargs_data={"image": ["encoded"]},
)
def test_multimodal_features_reject_overlapping_placeholders():
with pytest.raises(ValueError, match="non-overlapping"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={
"image": [
PlaceholderRangeInfo(offset=1, length=2),
PlaceholderRangeInfo(offset=2, length=2),
]
},
kwargs_data={"image": ["a", "b"]},
)
def test_generate_request_rejects_placeholder_outside_prompt():
with pytest.raises(ValueError, match="within the token_ids sequence"):
GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(),
features=MultiModalFeatures(
mm_hashes={"image": ["a"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
kwargs_data={"image": ["encoded"]},
),
)
def test_reasoning_parser_kwargs_rejects_unknown_keys():
"""The engine splats these into its reasoning parser's constructor, so a
client must not be able to pass arbitrary keyword arguments."""
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
with pytest.raises(ValidationError, match="tokenizer"):
GenerateRequest.model_validate(payload)
def test_rust_render_sampling_params_without_nulls_validate():
"""Rust render omits unset sampling fields; Python generate rejects nulls."""
sampling_params = {"max_tokens": 64, "watermarking": True}
request = GenerateRequest.model_validate(
{"token_ids": [1, 2, 3], "sampling_params": sampling_params}
)
assert request.sampling_params.max_tokens == 64
with pytest.raises(ValidationError, match="temperature"):
GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {**sampling_params, "temperature": None},
}
)
def test_output_mode_defaults_to_tokens():
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
def test_output_mode_text_is_accepted():
payload = {**_base_payload(), "output_mode": "text"}
assert GenerateRequest.model_validate(payload).output_mode == "text"
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
def test_output_mode_rejects_unsupported_levels(output_mode):
"""Levels the server does not implement fail instead of falling back to
tokens which would answer 200 with the wrong response shape."""
payload = {**_base_payload(), "output_mode": output_mode}
with pytest.raises(ValidationError, match="output_mode"):
GenerateRequest.model_validate(payload)
def test_text_mode_rejects_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "text",
}
with pytest.raises(ValidationError, match="detokenize"):
GenerateRequest.model_validate(payload)
def test_tokens_mode_allows_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "tokens",
}
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
def test_response_without_output_mode_parses_as_tokens():
"""Existing clients and older servers never send output_mode."""
parsed = TypeAdapter(GenerateResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensResponse)
assert parsed.output_mode == "tokens"
def test_stream_chunk_without_output_mode_parses_as_tokens():
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensStreamResponse)
def test_text_response_and_chunk_dispatch_on_output_mode():
response = TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
assert isinstance(response, GenerateTextResponse)
assert isinstance(chunk, GenerateTextStreamResponse)
def test_text_response_requires_text_on_every_choice():
with pytest.raises(ValidationError, match="text"):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0}]}
)
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
def test_response_rejects_unknown_output_mode(output_mode):
with pytest.raises(ValidationError):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": output_mode, "choices": []}
)
def test_derender_requests_accept_responses_without_output_mode():
DerenderChatRequest.model_validate(
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
)
DerenderChatStreamRequest.model_validate(
{
"stream": True,
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
}
)
def test_derender_requests_reject_text_responses():
"""Text responses are already detokenized with stop strings applied, so
re-decoding their token_ids would silently undo the truncation."""
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatRequest.model_validate({"generate_response": text})
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
def test_tokens_response_echoes_output_mode_without_text():
dumped = GenerateTokensResponse(
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
).model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for multimodal features through the /inference/v1/generate endpoint.
Mirrors test_serving_tokens.py but exercises the multimodal piping
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
/generate -> /detokenize path. Intentionally avoids running the HF
processor in the pytest parent process to keep os.fork() in other
tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free.
"""
import httpx
import pytest
import pytest_asyncio
from PIL import Image
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.utils import encode_image_url
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
GEN_ENDPOINT = "/inference/v1/generate"
RENDER_ENDPOINT = "/v1/chat/completions/render"
DETOKENIZE_ENDPOINT = "/detokenize"
@pytest.fixture(scope="module")
def test_image():
return Image.new("RGB", (224, 224), color=(255, 0, 0))
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
"--enable-scale-out",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_render_to_generate_roundtrip(client, test_image):
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
All preprocessing and detokenization happens in the server subprocess;
the pytest parent never imports transformers or touches torch tensors.
"""
data_url = encode_image_url(test_image, format="PNG")
render_payload = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{
"type": "text",
"text": "What color is this image? Answer in one word.",
},
],
}
],
}
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
render_resp.raise_for_status()
render_data = render_resp.json()
# Validate render output structure: keys exist and values are non-empty
# and well-typed.
assert "token_ids" in render_data
assert isinstance(render_data["token_ids"], list)
assert len(render_data["token_ids"]) > 0
assert all(isinstance(t, int) for t in render_data["token_ids"])
assert "features" in render_data
features = render_data["features"]
assert features is not None
assert isinstance(features, dict)
assert "mm_hashes" in features
assert "image" in features["mm_hashes"]
image_hashes = features["mm_hashes"]["image"]
assert isinstance(image_hashes, list)
assert len(image_hashes) > 0
assert all(isinstance(h, str) and h for h in image_hashes)
assert "mm_placeholders" in features
assert "image" in features["mm_placeholders"]
image_placeholders = features["mm_placeholders"]["image"]
assert isinstance(image_placeholders, list)
assert len(image_placeholders) > 0
for p in image_placeholders:
assert isinstance(p.get("offset"), int)
assert isinstance(p.get("length"), int)
assert p["length"] > 0
assert "kwargs_data" in features
assert "image" in features["kwargs_data"]
assert len(features["kwargs_data"]["image"]) > 0
# Build generate request from render output
generate_payload = render_data
generate_payload["sampling_params"] = {
"max_tokens": 10,
"temperature": 0.0,
}
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
assert isinstance(gen_data["choices"], list)
assert len(gen_data["choices"]) >= 1
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert isinstance(choice["token_ids"], list)
assert len(choice["token_ids"]) > 0
assert all(isinstance(t, int) for t in choice["token_ids"])
detok_resp = await client.post(
DETOKENIZE_ENDPOINT,
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
)
detok_resp.raise_for_status()
detok_data = detok_resp.json()
assert "prompt" in detok_data
text = detok_data["prompt"]
assert isinstance(text, str)
assert len(text) > 0
assert "red" in text.lower(), (
f"Expected model to identify the red image, got: {text!r}"
)
def _collapse_placeholders(
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
) -> list[int]:
"""Undo placeholder expansion: each expanded range becomes one token.
``/generate`` with ``content_parts`` expects unexpanded placeholders and
expands them itself, so the roundtrip must start from a collapsed prompt.
"""
ranges = sorted(
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
)
collapsed: list[int] = []
pos = 0
for offset, length in ranges:
collapsed.extend(token_ids[pos:offset])
collapsed.append(token_ids[offset])
pos = offset + length
collapsed.extend(token_ids[pos:])
return collapsed
async def _render_image_prompt(client, data_url: str) -> dict:
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What color is this?"},
],
}
],
},
)
render_resp.raise_for_status()
return render_resp.json()
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""Unexpanded prompt + content_parts must expand exactly like /render."""
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
assert len(unexpanded) < len(expanded)
gen_resp = await client.post(
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"return_token_ids": True,
},
)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
assert gen_data["prompt_token_ids"] == expanded
assert gen_data["mm_placeholders"] == mm_placeholders
@pytest.mark.asyncio
async def test_content_parts_streaming(client, test_image):
"""content_parts should work with streaming."""
import json as json_mod
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
async with client.stream(
"POST",
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
"stream": True,
"return_token_ids": True,
},
) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json_mod.loads(line[6:]))
assert len(chunks) > 0
prompt_metadata_chunks = [
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
]
assert len(prompt_metadata_chunks) == 1
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders
@@ -1,843 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
ABORT_ENDPOINT = "/inference/v1/abort_requests"
def get_vocab_size(model_name):
config = ModelConfig(
model=model_name,
seed=0,
dtype="bfloat16",
)
return config.get_vocab_size()
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(MODEL_NAME)
@pytest.fixture(scope="module")
def messages():
return [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "How many countries are in the EU?"},
]
@pytest.fixture(scope="module")
def server(request):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
# 1 ULP when the batch dimension (M) changes, because different M
# values cause the Tensile backend to select different tile
# configurations with different fp32 accumulation orders. With
# prefix caching, cache-miss prefills compute all tokens in one
# pass (large M) while cache-hit requests compute only the
# uncached suffix (small M), seeding a divergence that amplifies
# through the residual stream and flips argmax tokens.
# See: https://github.com/vllm-project/vllm/issues/33123
#
# Either disable prefix caching entirely, or enable it with
# --deterministic-prefix-caching which forces cache-miss prefills
# to split at block boundaries so the suffix GEMM shape is always
# identical regardless of cache state.
#
# Option A: disable prefix caching
"--no-enable-prefix-caching",
#
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
"--enable-scale-out",
]
extra_args = getattr(request, "param", None)
if extra_args is not None:
args = args + (
list(extra_args)
if isinstance(extra_args, (list, tuple))
else [str(extra_args)]
)
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_endpoint(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
assert data["choices"][0].get("sampling_mask") is None
@pytest.mark.asyncio
async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
"""Explicit null max_tokens must not skip the min_tokens bound."""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": None, "min_tokens": 2147483648},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "min_tokens" in resp.json()["error"]["message"]
followup = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=followup)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="sampling mask output is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"server",
[["--return-sampling-mask", "--logprobs-mode", "processed_logprobs"]],
indirect=True,
)
async def test_generate_sampling_mask(client):
top_k = 5
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.8,
"top_k": top_k,
"top_p": 0.9,
"ignore_eos": True,
"seed": 0,
"logprobs": top_k,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
choice = resp.json()["choices"][0]
token_ids = choice["token_ids"]
sampling_mask = choice["sampling_mask"]
assert sampling_mask is not None
assert len(token_ids) == len(sampling_mask)
vocab_size = get_vocab_size(MODEL_NAME)
logprobs_content = choice["logprobs"]["content"]
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
assert support
assert len(support) == len(set(support))
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
assert token_id in support
# processed_logprobs: exactly the support carries finite probability.
for top in entry["top_logprobs"]:
in_support = top["token_id"] in support
assert in_support == (top["logprob"] > -9999.0)
@pytest.mark.asyncio
async def test_generate_defaults_max_tokens_when_omitted(client):
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
defaulting was wired up, every request that didn't set ``max_tokens``
truncated mid-generation. The server should now fill it in from
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"temperature": 0.0,
"ignore_eos": True,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
completion_tokens = len(data["choices"][0]["token_ids"])
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
# should get ~1021 tokens of output (capped at max_model_len boundary).
assert completion_tokens > 16, (
f"expected server-side default to exceed the legacy 16-token cap, "
f"got {completion_tokens}"
)
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
)
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
"""Regression: ``n > 1`` must return ``n`` choices.
Non-streaming requests kept ``SamplingParams``' default output kind,
``CUMULATIVE``, so only the sequences updated during the last engine step
reached the response and the others were silently dropped.
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
@pytest.mark.asyncio
async def test_generate_stream(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": True,
}
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
payload_str = line[len("data: ") :]
if payload_str == "[DONE]":
break
chunks.append(json.loads(payload_str))
assert len(chunks) > 0
# Every chunk has choices with token_ids
all_token_ids = []
for chunk in chunks:
assert "choices" in chunk
assert len(chunk["choices"]) == 1
choice = chunk["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
all_token_ids.extend(choice["token_ids"])
# Last chunk should have a finish_reason
assert chunks[-1]["choices"][0]["finish_reason"] is not None
# Streaming should produce the same tokens as non-streaming
non_stream_resp = await client.post(
GEN_ENDPOINT,
json={
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
"stream": False,
},
)
non_stream_data = non_stream_resp.json()
# Just verify we got the right number of tokens
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
@pytest.mark.asyncio
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
async def test_generate_logprobs(client, logprobs_value):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.0,
"logprobs": logprobs_value,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["logprobs"] is not None
logprobs_content = choice["logprobs"]["content"]
assert len(logprobs_content) == len(choice["token_ids"])
for entry in logprobs_content:
assert "logprob" in entry
assert len(entry["top_logprobs"]) >= 1
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
for ignore_eos in [True, False]:
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
# NOTE coordinator will set this to skip detokenization
"detokenize": False,
"ignore_eos": ignore_eos,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
gen_token_ids = generate_data["choices"][0]["token_ids"]
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"ignore_eos": ignore_eos,
"chat_template_kwargs": {"enable_thinking": False},
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
if ignore_eos:
# When ignoring EOS, only compare up to the first EOS token
# Post-EOS generation is undefined and may differ
eos_tokens = {
tokenizer.eos_token_id,
*tokenizer.extra_special_tokens_ids,
}
# Find first EOS in generated tokens
eos_pos = None
for i, tid in enumerate(gen_token_ids):
if tid in eos_tokens:
eos_pos = i
break
if eos_pos is not None:
gen_token_ids_truncated = gen_token_ids[:eos_pos]
generate_res = tokenizer.decode(
gen_token_ids_truncated, skip_special_tokens=True
)
# Truncate completions_res to same length for comparison
completions_res = completions_res[: len(generate_res)]
assert generate_res == completions_res
@pytest.mark.asyncio
async def test_stop_string_workflow(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
# stop strings are only supported when detokenize is True.
"stop": ["27 member"],
},
# TODO stream test is much more interesting
"stream": False,
}
with pytest.raises(httpx.HTTPStatusError):
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_resp.raise_for_status()
payload["sampling_params"]["stop"] = None
generate_resp = await client.post(
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
# NOTE This is under the responsibility of the coordinator
# stop_checker = StopChecker(
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
# )
stop_str, truncate_to = check_stop_strings(
generate_res, len(generate_res), ["27 member"], False
)
assert stop_str == "27 member"
# abort request that hit stop string (requires tokens-only mode)
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
# res.raise_for_status()
generate_res = generate_res[:truncate_to]
# Get stop_str response from chat completions
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"stop": ["27 member"],
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_stop_strings_rejected_when_tokens_only(client):
"""--tokens-only forces detokenize=False after request validation, so
stop strings must be rejected explicitly rather than silently ignored."""
sampling_params = {"max_tokens": 5, "stop": ["never"]}
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": sampling_params,
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "stop strings" in resp.json()["error"]["message"]
sampling_params["stop"] = None
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server",
[
[
"--enable-lora",
"--lora-modules",
"Alice=charent/self_cognition_Alice",
"Bob=charent/self_cognition_Bob",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
],
indirect=True,
)
async def test_generate_with_lora_adapter(client, tokenizer, messages):
# Verify adapters are listed
models_resp = await client.get("/v1/models")
models_resp.raise_for_status()
models = {m["id"] for m in models_resp.json().get("data", [])}
assert {"Alice", "Bob"}.issubset(models)
# Generate using a LoRA adapter by specifying its name as the model
payload = {
"model": "Alice",
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": "Alice",
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
payload = {
"model": "Alice",
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
def _structured_chat_body(messages, **overrides) -> dict:
return {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 64,
"temperature": 0.0,
"chat_template_kwargs": {"enable_thinking": True},
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"count": {"type": "integer"}},
"required": ["count"],
},
},
},
**overrides,
}
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
render_resp = await client.post("/v1/chat/completions/render", json=body)
render_resp.raise_for_status()
generate_request = render_resp.json()
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
generate_resp.raise_for_status()
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
# <think> tokens, so the engine gates structured outputs differently when
# either reasoning field is dropped between render and generate.
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
"""With thinking on, structured outputs must wait for the reasoning to
end on generate, the same as on chat."""
body = _structured_chat_body(messages, return_token_ids=True)
chat_resp = await client.post("/v1/chat/completions", json=body)
chat_resp.raise_for_status()
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is False
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_ended(
client, tokenizer, messages
):
"""`include_reasoning=false` constrains from the first token on chat, so
generate must too, even though the engine's parser expects thinking."""
body = _structured_chat_body(messages, include_reasoning=False)
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is True
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
assert isinstance(output["count"], int)
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
return tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
def _text_mode_payload(token_ids, **sampling_params) -> dict:
return {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
"output_mode": "text",
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
resp.raise_for_status()
data = resp.json()
assert data["output_mode"] == "text"
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": {"enable_thinking": False},
}
chat_resp = await client.post("/v1/chat/completions", json=payload)
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
assert data["choices"][0]["text"] == chat_content
assert data["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
payload = _text_mode_payload(token_ids)
non_stream = await client.post(GEN_ENDPOINT, json=payload)
non_stream.raise_for_status()
chunks = []
async with client.stream(
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json.loads(line[len("data: ") :]))
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert chunks[-1]["choices"][0]["finish_reason"] is not None
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
assert streamed_text == non_stream.json()["choices"][0]["text"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"overrides, expected",
[
pytest.param(
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
),
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
],
)
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
payload = {
**_text_mode_payload([1, 2, 3], max_tokens=5),
**overrides,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert expected in resp.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_text_mode_rejected_when_tokens_only(client):
"""Without a tokenizer the detokenizer returns "", so the request must
fail instead of answering 200 with empty text."""
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
assert resp.status_code == 400
assert "requires a tokenizer" in resp.json()["error"]["message"]
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
payload["sampling_params"]["detokenize"] = False
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
async def test_stream_delivers_abort_finish_chunk(
client, tokenizer, messages, output_mode
):
"""The final output after an abort has no new token IDs, but the
stream must still deliver its finish reason."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"output_mode": output_mode,
"stream": True,
"request_id": f"{output_mode}-abort-e2e",
}
chunks = []
aborted = False
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
)
assert abort.status_code == 200
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
assert generated < 900
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
resp = await client.post(
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 200
resp = await client.post(ABORT_ENDPOINT, json={})
assert resp.status_code == 400
# The unauthenticated path is only served with --tokens-only.
resp = await client.post(
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 404
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from pydantic import ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own integer token id."""
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
)
token_ids = {e.token_id for e in result.content[0].top_logprobs}
assert token_ids == {262, 257}, f"got {token_ids}"
def test_sampled_entry_carries_token_id_and_rank():
"""The sampled token is identified by id, and the engine's rank survives."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
],
)
entry = result.content[0]
assert entry.token_id == 262
assert entry.logprob == -0.1
assert entry.rank == 1
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
# No tokenizer on the generate server: no string token, no bytes.
assert not hasattr(entry, "token")
assert not hasattr(entry, "bytes")
def test_sampled_token_absent_from_topk_uses_sentinel():
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[5],
top_logprobs=[{7: Logprob(-0.9)}],
)
entry = result.content[0]
assert entry.token_id == 5
assert entry.logprob == -9999.0
assert entry.rank is None
assert entry.top_logprobs == []
def test_logprobs_zero_emits_sampled_token():
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9)}],
)
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
def test_logprobs_minus_one_emits_all_tokens():
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
assert len(result.content[0].top_logprobs) == 2
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
entry = result.content[0]
assert entry.token == "token_id:7"
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
tokenizer=tokenizer,
)
entry = result.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
tokenizer.decode.assert_not_called()
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")
def test_sampled_token_outside_topk_comes_first():
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
all k + 1 candidates when the sampled token is outside the top k; derender
applies each endpoint's cut (#59513)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[50],
top_logprobs=[
{
50: Logprob(-3.0, rank=5),
10: Logprob(-0.2, rank=1),
20: Logprob(-1.0, rank=2),
}
],
)
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
def test_logprob_is_required_on_the_wire():
"""A payload missing logprob is rejected rather than read as -9999."""
with pytest.raises(ValidationError):
GenerateLogProb.model_validate({"token_id": 1})
def test_rank_zero_from_a_nan_logprob_is_none():
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
)
entry = result.content[0]
assert entry.rank is None
assert entry.top_logprobs[0].rank is None
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
def test_text_logprobs_clamp_a_nan_logprob():
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
tokenizer=MagicMock(),
)
entry = result.content[0]
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
@@ -120,7 +120,7 @@ class TestAffectedModulesUseSanitize:
@pytest.mark.parametrize(
"module",
[
"vllm.entrypoints.speech_to_text.realtime.connection",
"vllm.entrypoints.serve.exception_handling.handlers.http",
],
)
def test_module_calls_sanitize_message(self, module: str):
@@ -1,371 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
import shutil
from contextlib import suppress
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
BADREQUEST_CASES = [
(
"test_rank",
{"r": 1024},
"is greater than max_lora_rank",
),
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
(
"test_modules_to_save",
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
]
@pytest.fixture(scope="module", params=[True])
def server_with_lora_modules_json(request, qwen3_lora_files):
# Define the json format LoRA module configurations
lora_module_1 = {
"name": "qwen3-lora",
"path": qwen3_lora_files,
"base_model_name": MODEL_NAME,
}
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
json.dumps(lora_module_1),
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"64",
]
# Enable the /v1/load_lora_adapter endpoint
envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server_with_lora_modules_json):
async with server_with_lora_modules_json.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert served_model.parent is None
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
@pytest.mark.asyncio
async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
)
# Ensure adapter loads before querying /models
assert "success" in response
models = await client.models.list()
models = models.data
dynamic_lora_model = models[-1]
assert dynamic_lora_model.root == qwen3_lora_files
assert dynamic_lora_model.parent == MODEL_NAME
assert dynamic_lora_model.id == "qwen3-lora-3"
@pytest.mark.asyncio
async def test_load_lora_adapter_with_same_name_replaces_inplace(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
):
"""Test that loading a LoRA adapter with the same name replaces it inplace."""
adapter_name = "replaceable-adapter"
messages = [
{"content": "Follow the instructions to make animal noises", "role": "system"},
{"content": "Make your favorite animal noise.", "role": "user"},
]
# Load LoRA that makes model meow
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Meow Meow Meow" in completion.choices[0].message.content
# Load LoRA that makes model woof
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_woofing_lora_files,
"load_inplace": True,
},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Woof Woof Woof" in completion.choices[0].message.content
@pytest.mark.asyncio
async def test_load_lora_adapter_with_load_inplace_false_errors(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files
):
"""Test that load_inplace=False returns an error when adapter already exists."""
adapter_name = "test-load-inplace-false"
# Load LoRA adapter first time (should succeed)
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
# Try to load the same adapter again with load_inplace=False (should fail)
with pytest.raises(openai.BadRequestError) as exc_info:
await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_meowing_lora_files,
},
)
# Verify the error message
assert "already been loaded" in str(exc_info.value)
@pytest.mark.asyncio
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
with pytest.raises(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
@pytest.mark.asyncio
async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
with pytest.raises(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid-json", "lora_path": str(invalid_files)},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES)
async def test_dynamic_lora_badrequests(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
test_name: str,
config_change: dict,
expected_error: str,
):
# Create test directory
test_dir = tmp_path / test_name
# Copy adapter files
shutil.copytree(qwen3_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(config_change)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
# Test loading the adapter
with pytest.raises(openai.InternalServerError, match=expected_error):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": test_name, "lora_path": str(test_dir)},
)
@pytest.mark.asyncio
async def test_multiple_lora_adapters(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
"""Validate that many loras can be dynamically registered and inferenced
with concurrently"""
# This test file configures the server with --max-cpu-loras=2 and this test
# will concurrently load 10 adapters, so it should flex the LRU cache
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
lora_tasks = []
for i in range(10):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
@pytest.mark.asyncio
async def test_loading_invalid_adapters_does_not_break_others(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
stop_good_requests_event = asyncio.Event()
async def run_good_requests(client):
# Run chat completions requests until event set
results = []
while not stop_good_requests_event.is_set():
try:
batch = await client.completions.create(
model="qwen3-lora",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
results.append(batch)
except Exception as e:
results.append(e)
return results
# Create task to run good requests
good_task = asyncio.create_task(run_good_requests(client))
# Run a bunch of bad adapter loads
for _ in range(25):
with suppress(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
for _ in range(25):
with suppress(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid", "lora_path": str(invalid_files)},
)
# Ensure all the running requests with lora adapters succeeded
stop_good_requests_event.set()
results = await good_task
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
# Ensure we can load another adapter and run it
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "valid", "lora_path": qwen3_lora_files},
)
await client.completions.create(
model="valid",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
@pytest.mark.asyncio
async def test_beam_search_with_lora_adapters(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
):
"""Validate that async beam search can be used with lora."""
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
extra_body=dict(use_beam_search=True),
)
lora_tasks = []
for i in range(3):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
@@ -214,7 +214,6 @@ def test_request_logger_log_outputs_integration():
prompt_token_ids=[1, 2, 3],
prompt_embeds=None,
params=None,
lora_request=None,
)
request_logger.log_outputs(
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.audio import AudioAsset
def add_attention_backend(server_args, attention_config):
"""Append attention backend CLI arg if specified.
Args:
server_args: List of server arguments to extend in-place.
attention_config: Dict with 'backend' key, or None.
"""
if attention_config and "backend" in attention_config:
server_args.extend(["--attention-backend", attention_config["backend"]])
@pytest.fixture
def mary_had_lamb():
path = AudioAsset("mary_had_lamb").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def winning_call():
path = AudioAsset("winning_call").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def foscolo():
# Test translation it->en
path = AudioAsset("azacinto_foscolo").get_local_path()
with open(str(path), "rb") as f:
yield f
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
@pytest.fixture(scope="module")
def transcription_server_with_force_include_usage():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-num-seqs",
"4",
"--enforce-eager",
"--enable-force-include-usage",
"--gpu-memory-utilization",
"0.2",
]
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def transcription_client_with_force_include_usage(
transcription_server_with_force_include_usage,
):
async with (
transcription_server_with_force_include_usage.get_async_client() as async_client
):
yield async_client
@pytest.mark.asyncio
async def test_transcription_with_enable_force_include_usage(
transcription_client_with_force_include_usage, winning_call
):
res = (
await transcription_client_with_force_include_usage.audio.transcriptions.create(
model="openai/whisper-large-v3-turbo",
file=winning_call,
language="en",
temperature=0.0,
stream=True,
timeout=30,
)
)
async for chunk in res:
if not len(chunk.choices):
# final usage sent
usage = chunk.usage
assert isinstance(usage, dict)
assert usage["prompt_tokens"] > 0
assert usage["completion_tokens"] > 0
assert usage["total_tokens"] > 0
else:
assert not hasattr(chunk, "usage")
@@ -592,13 +592,12 @@ class TestLoadWeightsExpertMapping:
return calls
@pytest.mark.parametrize(
"projs,lora_prefix",
"projs",
[
(("gate_proj", "down_proj", "up_proj"), ""),
(("w1", "w2", "w3"), ""),
(("up_proj", "down_proj", "up_proj"), ""),
(("up_proj", "down_proj", None), ""),
(("gate_proj", "down_proj", "up_proj"), "base_layer."),
("gate_proj", "down_proj", "up_proj"),
("w1", "w2", "w3"),
("up_proj", "down_proj", "up_proj"),
("up_proj", "down_proj", None),
],
)
@pytest.mark.parametrize(
@@ -610,12 +609,11 @@ class TestLoadWeightsExpertMapping:
],
)
def test_per_expert_weights_keep_all_physical_destinations(
self, projs, lora_prefix, suffix, shape, dtype
self, projs, suffix, shape, dtype
):
mapping = self._mapping(
projs,
num_redundant_experts=4,
lora_base_layer_prefix=lora_prefix,
is_gated=projs[2] is not None,
)
for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]:
@@ -623,7 +621,7 @@ class TestLoadWeightsExpertMapping:
if not proj:
continue
weight = torch.full(shape, logical_id + 1, dtype=dtype)
name = f"{logical_id}.{proj}.{lora_prefix}{suffix}"
name = f"{logical_id}.{proj}.{suffix}"
calls = self._load(mapping, [(name, weight)], suffix)
shards = [s for p, s in zip(projs, ("w1", "w2", "w3")) if p == proj]
expected = [
@@ -636,20 +634,18 @@ class TestLoadWeightsExpertMapping:
torch.testing.assert_close(call[3], weight, rtol=0, atol=0)
@pytest.mark.parametrize(
"projs,fused_name,lora_prefix",
"projs,fused_name",
[
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj", ""),
(("w1", "w2", "w3"), "w13", "base_layer."),
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj"),
(("w1", "w2", "w3"), "w13"),
],
)
def test_per_expert_fused_gate_up_keeps_both_halves(
self, projs, fused_name, lora_prefix
self, projs, fused_name
):
mapping = self._mapping(
projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix
)
mapping = self._mapping(projs, num_redundant_experts=4)
weight = torch.arange(12).reshape(4, 3)
calls = self._load(mapping, [(f"1.{fused_name}.{lora_prefix}weight", weight)])
calls = self._load(mapping, [(f"1.{fused_name}.weight", weight)])
assert [call[:3] for call in calls] == [
("w13_weight", shard, expert)
for expert in (1, 17)
View File
-357
View File
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections import OrderedDict
from importlib import reload
from unittest.mock import MagicMock
import pytest
import torch
import torch.nn as nn
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.interfaces import SupportsLoRA
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@pytest.fixture()
def should_do_global_cleanup_after_test(request) -> bool:
"""Allow subdirectories to skip global cleanup by overriding this fixture.
This can provide a ~10x speedup for non-GPU unit tests since they don't need
to initialize torch.
"""
return not request.node.get_closest_marker("skip_global_cleanup")
@pytest.fixture(autouse=True)
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
yield
if should_do_global_cleanup_after_test:
cleanup_dist_env_and_memory(shutdown_ray=True)
@pytest.fixture
def maybe_enable_lora_dual_stream(monkeypatch: pytest.MonkeyPatch):
if current_platform.is_cuda():
monkeypatch.setenv("VLLM_LORA_ENABLE_DUAL_STREAM", "1")
import vllm.lora.layers.base_linear
if not hasattr(vllm.lora.layers.base_linear, "lora_linear_async"):
# Reload the module to ensure the environment variable takes effect.
reload(vllm.lora.layers.base_linear)
yield
@pytest.fixture
def dist_init():
from tests.utils import ensure_current_vllm_config
temp_file = tempfile.mkstemp()[1]
backend = "gloo" if current_platform.is_tpu() else current_platform.dist_backend
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend=backend,
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory(shutdown_ray=True)
class DummyLoRAModel(nn.Sequential, SupportsLoRA):
pass
@pytest.fixture
def dummy_model(default_vllm_config) -> nn.Module:
model = DummyLoRAModel(
OrderedDict(
[
("dense1", ColumnParallelLinear(764, 100)),
("dense2", RowParallelLinear(100, 50)),
(
"layer1",
nn.Sequential(
OrderedDict(
[
("dense1", ColumnParallelLinear(100, 10)),
("dense2", RowParallelLinear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("output", ColumnParallelLinear(50, 10)),
("outact", nn.Sigmoid()),
# Special handling for lm_head & sampler
("lm_head", ParallelLMHead(32064, 10)),
("logits_processor", LogitsProcessor(32064)),
]
)
)
model.config = MagicMock()
model.embedding_modules = {"lm_head": "lm_head"}
model.unpadded_vocab_size = 32064
return model
@pytest.fixture
def dummy_model_gate_up(default_vllm_config) -> nn.Module:
model = DummyLoRAModel(
OrderedDict(
[
("dense1", ColumnParallelLinear(764, 100)),
("dense2", RowParallelLinear(100, 50)),
(
"layer1",
nn.Sequential(
OrderedDict(
[
("dense1", ColumnParallelLinear(100, 10)),
("dense2", RowParallelLinear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("gate_up_proj", MergedColumnParallelLinear(50, [5, 5])),
("outact", nn.Sigmoid()),
# Special handling for lm_head & sampler
("lm_head", ParallelLMHead(32064, 10)),
("logits_processor", LogitsProcessor(32064)),
]
)
)
model.config = MagicMock()
model.packed_modules_mapping = {
"gate_up_proj": [
"gate_proj",
"up_proj",
],
}
model.embedding_modules = {"lm_head": "lm_head"}
model.unpadded_vocab_size = 32064
return model
@pytest.fixture(scope="session")
def mixtral_lora_files():
# Note: this module has incorrect adapter_config.json to test
# https://github.com/vllm-project/vllm/pull/5909/files.
return hf_api().snapshot_download(repo_id="SangBinCho/mixtral-lora")
@pytest.fixture(scope="session")
def chatglm3_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
@pytest.fixture(scope="session")
def baichuan_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
@pytest.fixture(scope="session")
def baichuan_zero_lora_files():
# all the lora_B weights are initialized to zero.
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
@pytest.fixture(scope="session")
def baichuan_regex_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan-7b-lora-zero-regex")
@pytest.fixture(scope="session")
def ilama_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/ilama-text2sql-spider")
@pytest.fixture(scope="session")
def minicpmv_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
@pytest.fixture(scope="session")
def qwen2vl_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
@pytest.fixture(scope="session")
def qwen25vl_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen25-vl-lora-pokemon")
@pytest.fixture(scope="session")
def qwen2vl_language_lora_files():
return hf_api().snapshot_download(
repo_id="prashanth058/qwen2vl-flickr-lora-language"
)
@pytest.fixture(scope="session")
def qwen2vl_vision_tower_connector_lora_files():
return hf_api().snapshot_download(
repo_id="prashanth058/qwen2vl-flickr-lora-tower-connector"
)
@pytest.fixture(scope="session")
def qwen2vl_vision_tower_lora_files():
return hf_api().snapshot_download(repo_id="prashanth058/qwen2vl-flickr-lora-tower")
@pytest.fixture(scope="session")
def qwen25vl_vision_lora_files():
return hf_api().snapshot_download(
repo_id="EpochEcho/qwen2.5-3b-vl-lora-vision-connector"
)
@pytest.fixture(scope="session")
def qwen3vl_vision_lora_files():
return hf_api().snapshot_download(
repo_id="EpochEcho/qwen3-4b-vl-lora-vision-connector"
)
@pytest.fixture(scope="session")
def gemma4_vision_lora_files():
return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 Meow LoRA files once per test session."""
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
@pytest.fixture(scope="session")
def qwen3_woofing_lora_files():
"""Download Qwen3 Woof LoRA files once per test session."""
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def qwen3_guard_star_trek_lora_files():
return hf_api().snapshot_download(
repo_id="geoffmunn/Qwen3Guard-StarTrek-Classification-0.6B",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def qwen3_guard_new_zealand_lora_files():
return hf_api().snapshot_download(
repo_id="geoffmunn/Qwen3Guard-NewZealand-Classification-0.6B",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def skywork_qwen3_reward_lora_files():
return hf_api().snapshot_download(
repo_id="AmirMohseni/skywork-qwen3-0.6b-reward-lora",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def deepseekv2_lora_files():
return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA")
@pytest.fixture(scope="session")
def gptoss20b_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/gpt-oss-20b-lora-adapter-text2sql"
)
@pytest.fixture(scope="session")
def qwen3moe_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen3-moe-text2sql-spider")
@pytest.fixture(scope="session")
def olmoe_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/olmoe-instruct-text2sql-spider")
@pytest.fixture(scope="session")
def qwen3_lora_files():
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
@pytest.fixture(scope="session")
def llama32_lora_huggingface_id():
# huggingface repo id is used to test lora runtime downloading.
return "jeeejeee/llama32-3b-text2sql-spider"
@pytest.fixture(scope="session")
def llama32_lora_files(llama32_lora_huggingface_id):
return hf_api().snapshot_download(repo_id=llama32_lora_huggingface_id)
@pytest.fixture(scope="session")
def whisper_lora_files():
return hf_api().snapshot_download(
repo_id="chengyili2005/whisper-small-mandarin-lora"
)
@pytest.fixture(scope="session")
def qwen35_text_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen35-4b-text-only-sql-lora")
@pytest.fixture(scope="session")
def qwen35_vl_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora"
)
@pytest.fixture(scope="session")
def qwen36_moe_2d_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen36-35ba3b-2d-weights-poken-lora"
)
@pytest.fixture(scope="session")
def qwen36_moe_3d_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora"
)
@pytest.fixture
def reset_default_device():
"""Some tests, such as `test_punica_ops.py`, explicitly set the
default device, which can affect subsequent tests. Adding this fixture
helps avoid this problem.
"""
original_device = torch.get_default_device()
yield
torch.set_default_device(original_device)
-39
View File
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
from vllm.lora.layers.utils import _get_lora_device
pytestmark = pytest.mark.skip_global_cleanup
def _param() -> nn.Parameter:
return nn.Parameter(torch.empty(1), requires_grad=False)
def test_get_lora_device_unquantized():
base_layer = nn.Module()
base_layer.weight = _param()
assert _get_lora_device(base_layer) == base_layer.weight.device
def test_get_lora_device_gptq_awq():
base_layer = nn.Module()
base_layer.qweight = _param()
assert _get_lora_device(base_layer) == base_layer.qweight.device
def test_get_lora_device_ark_linear():
base_layer = nn.Module()
base_layer.ark_linear = nn.Module()
base_layer.ark_linear.qweight = _param()
assert _get_lora_device(base_layer) == base_layer.ark_linear.qweight.device
def test_get_lora_device_unsupported_raises():
base_layer = nn.Module()
with pytest.raises(ValueError, match="Unsupported base layer"):
_get_lora_device(base_layer)
-114
View File
@@ -1,114 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Script to test add_lora, remove_lora, pin_lora, list_loras functions."""
import pytest
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
from vllm.entrypoints.launchers.api_server.entry import (
build_async_engine_client_from_engine_args,
)
from vllm.lora.request import LoRARequest
from vllm.v1.engine.llm_engine import LLMEngine
MODEL_PATH = "Qwen/Qwen3-0.6B"
LORA_MODULE_PATH = "charent/self_cognition_Alice"
LORA_RANK = 8
def make_lora_request(lora_id: int):
return LoRARequest(
lora_name=f"{lora_id}", lora_int_id=lora_id, lora_path=LORA_MODULE_PATH
)
def test_lora_functions_sync():
max_loras = 4
# Create engine in eager-mode. Due to high max_loras, the CI can
# OOM during cuda-graph capture.
engine_args = EngineArgs(
model=MODEL_PATH,
enable_lora=True,
max_loras=max_loras,
max_lora_rank=LORA_RANK,
max_model_len=128,
gpu_memory_utilization=0.8,
enforce_eager=True,
)
llm = LLMEngine.from_engine_args(engine_args)
def run_check(fn, args, expected: list):
fn(args)
assert set(llm.list_loras()) == set(expected)
run_check(llm.add_lora, make_lora_request(1), [1])
run_check(llm.add_lora, make_lora_request(2), [1, 2])
# Pin LoRA 1 and test that it is never removed on subsequent adds.
run_check(llm.pin_lora, 1, [1, 2])
run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
# Remove LoRA 1 and continue adding.
run_check(llm.remove_lora, 1, [8, 9, 10])
run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
# Remove all LoRAs.
run_check(llm.remove_lora, 13, [12, 10, 11])
run_check(llm.remove_lora, 12, [10, 11])
run_check(llm.remove_lora, 11, [10])
run_check(llm.remove_lora, 10, [])
@pytest.mark.asyncio
async def test_lora_functions_async():
max_loras = 4
engine_args = AsyncEngineArgs(
model=MODEL_PATH,
enable_lora=True,
max_loras=max_loras,
max_lora_rank=LORA_RANK,
max_model_len=128,
gpu_memory_utilization=0.8,
enforce_eager=True,
)
async def run_check(fn, args, expected: list):
await fn(args)
assert set(await llm.list_loras()) == set(expected)
async with build_async_engine_client_from_engine_args(engine_args) as llm:
await run_check(llm.add_lora, make_lora_request(1), [1])
await run_check(llm.add_lora, make_lora_request(2), [1, 2])
# Pin LoRA 1 and test that it is never removed on subsequent adds.
await run_check(llm.pin_lora, 1, [1, 2])
await run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
await run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
await run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
await run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
await run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
await run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
await run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
await run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
# Remove LoRA 1 and continue adding.
await run_check(llm.remove_lora, 1, [8, 9, 10])
await run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
await run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
await run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
# Remove all LoRAs
await run_check(llm.remove_lora, 13, [12, 10, 11])
await run_check(llm.remove_lora, 12, [10, 11])
await run_check(llm.remove_lora, 11, [10])
await run_check(llm.remove_lora, 10, [])
-51
View File
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.lora.utils import is_in_target_modules
class TestIsInTargetModules:
"""Tests for is_in_target_modules (deployment-time filter)."""
def test_none_allows_all(self):
assert is_in_target_modules("model.layers.0.self_attn.o_proj", None)
def test_suffix_in_target(self):
assert is_in_target_modules(
"model.layers.0.self_attn.o_proj", ["o_proj", "q_proj"]
)
def test_suffix_not_in_target(self):
assert not is_in_target_modules(
"model.layers.0.self_attn.o_proj", ["q_proj", "k_proj"]
)
def test_empty_target_modules(self):
assert not is_in_target_modules("model.layers.0.self_attn.o_proj", [])
def test_exact_name_match(self):
assert is_in_target_modules("dense1", ["dense1", "dense2"])
def test_exact_name_no_match(self):
assert not is_in_target_modules("dense3", ["dense1", "dense2"])
def test_packed_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_up_proj",
["gate_proj", "up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_packed_child_matches_parent_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_proj",
["gate_up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_fused_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.self_attn.fused_qkv_a_proj",
["q_a_proj", "kv_a_proj_with_mqa"],
{"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"]},
)
-109
View File
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
RANK, ALPHA = 16, 32
# PEFTHelper stores alpha/sqrt(r) for use_rslora adapters.
RSLORA_SCALING = ALPHA / RANK**0.5
@pytest.fixture(autouse=True)
def should_do_global_cleanup_after_test():
# Pure-python weight packing; skip GPU/ray cleanup.
return False
def moe_loras(scaling: float, stacked: bool = False) -> list[LoRALayerWeights]:
a_shape = (3, RANK, 4) if stacked else (RANK, 4)
b_shape = (3, 6, RANK) if stacked else (6, RANK)
return [
LoRALayerWeights(
module_name=f"experts.w{i}",
rank=RANK,
lora_alpha=ALPHA,
lora_a=torch.randn(*a_shape),
lora_b=torch.randn(*b_shape),
scaling=scaling,
)
for i in (1, 2, 3)
]
@pytest.mark.parametrize(
"packer,stacked",
[
(PackedLoRALayerWeights.pack_moe, False),
(PackedLoRALayerWeights.pack_moe_stacked, True),
],
)
def test_moe_packing_uses_stored_scaling(packer, stacked: bool):
packed = packer(moe_loras(RSLORA_SCALING, stacked), "experts")
assert packed.scaling == pytest.approx([RSLORA_SCALING] * 3)
def test_non_gated_moe_keeps_w3_unscaled():
packed = PackedLoRALayerWeights.pack_moe(
moe_loras(RSLORA_SCALING), "experts", is_non_gated_moe=True
)
assert packed.scaling[:2] == pytest.approx([RSLORA_SCALING] * 2)
assert packed.scaling[2] == 1.0
@pytest.mark.parametrize(
"packer,stacked",
[
(PackedLoRALayerWeights.pack_moe, False),
(PackedLoRALayerWeights.pack_moe_stacked, True),
],
)
def test_moe_packing_keeps_per_projection_scaling(packer, stacked: bool):
# e.g. rank_pattern / alpha_pattern giving gate, down and up different scalings
loras = moe_loras(1.0, stacked)
for lora, scaling in zip(loras, (2.0, 4.0, 8.0)):
lora.scaling = scaling
packed = packer(loras, "experts")
assert packed.scaling == pytest.approx([2.0, 4.0, 8.0])
def test_moe_packing_keeps_per_expert_scaling():
"""Experts of one projection can have different scaling, e.g. from
alpha_pattern."""
loras = moe_loras(RSLORA_SCALING) + moe_loras(RSLORA_SCALING)
loras[4].scaling = 2 * RSLORA_SCALING # w2 of expert 1
expected = [lora.lora_b * lora.scaling for lora in loras]
packed = PackedLoRALayerWeights.pack_moe(loras, "experts").optimize()
for i, lora_b in enumerate(packed.lora_b):
torch.testing.assert_close(lora_b, torch.stack(expected[i::3]))
def test_moe_packing_rejects_per_expert_rank():
"""Experts of one projection must share a rank to be stacked."""
loras = moe_loras(1.0) + moe_loras(1.0)
loras[4].lora_a = torch.randn(RANK // 2, 4)
loras[4].lora_b = torch.randn(6, RANK // 2)
with pytest.raises(ValueError, match="different LoRA ranks"):
PackedLoRALayerWeights.pack_moe(loras, "experts")
def test_from_config_uses_checkpoint_module_name():
peft_helper = PEFTHelper(
r=RANK,
lora_alpha=ALPHA,
target_modules=["q_proj"],
rank_pattern={r"^model\.layers\.0\.self_attn\.q_proj": 4},
)
# the pattern refers to the checkpoint name, not the name after weights mapping
lora = LoRALayerWeights.from_config(
"language_model.model.layers.0.self_attn.q_proj",
peft_helper,
"model.layers.0.self_attn.q_proj",
)
assert lora.rank == 4
assert lora.scaling == pytest.approx(ALPHA / 4)
lora = LoRALayerWeights.from_config("model.layers.1.self_attn.q_proj", peft_helper)
assert lora.rank == RANK
assert lora.scaling == pytest.approx(ALPHA / RANK)
-89
View File
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end correctness test for 2D MoE LoRA expert-parallel
load-time slicing
"""
import pytest
import torch
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
from vllm.lora.peft_helper import PEFTHelper
NUM_LAYERS = 48
GLOBAL_NUM_EXPERTS = 128
LOCAL_NUM_EXPERTS = 64 # ep_size = 2
EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
def _expected_lora_modules() -> set[str]:
"""Replicate the set ``WorkerLoRAManager._load_adapter`` would build
from this model's ``packed_modules_mapping``."""
expected: set[str] = set(NON_EXPERT_MODULES)
for expert in range(GLOBAL_NUM_EXPERTS):
for proj in EXPERT_PROJECTIONS:
expected.add(f"experts.{expert}.{proj}")
return expected
def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
return LoRAModel.from_local_checkpoint(
lora_dir,
_expected_lora_modules(),
peft_helper=peft_helper,
lora_model_id=lora_id,
device="cpu",
moe_ep_spec=moe_ep_spec,
)
@pytest.mark.parametrize("ep_rank", [0, 1])
def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
"""ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
LoRA has the right size, the right expert membership, and the
right tensor values."""
peft_helper = PEFTHelper.from_local_dir(
qwen3moe_lora_files, max_position_embeddings=4096
)
# Baseline: no spec → loads every expert × projection × layer plus
# all non-expert LoRA modules.
ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
expected_baseline = (
GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(ground_truth.loras) == expected_baseline
# Sliced load: only this rank's experts; non-expert LoRA is untouched.
spec = MoEEPLoadSpec(
ep_rank=ep_rank,
local_num_experts=LOCAL_NUM_EXPERTS,
global_num_experts=GLOBAL_NUM_EXPERTS,
)
sliced = _load(
qwen3moe_lora_files,
peft_helper,
moe_ep_spec=spec,
lora_id=100 + ep_rank,
)
expected_sliced = (
LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(sliced.loras) == expected_sliced
expert_start = ep_rank * LOCAL_NUM_EXPERTS
expert_end = expert_start + LOCAL_NUM_EXPERTS
for name, lora in sliced.loras.items():
gt = ground_truth.loras[name]
torch.testing.assert_close(lora.lora_a, gt.lora_a)
torch.testing.assert_close(lora.lora_b, gt.lora_b)
if ".experts." in name:
expert_idx = int(name.split(".experts.")[-1].split(".")[0])
assert expert_start <= expert_idx < expert_end, (
f"non-local expert {expert_idx} leaked: {name}"
)
-260
View File
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
import shutil
import pytest
from vllm.config.lora import LoRAConfig
from vllm.lora.peft_helper import PEFTHelper
ERROR_CASES = [
(
"test_rank",
{"r": 1024},
"is greater than max_lora_rank",
),
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
(
"test_modules_to_save",
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
(
"test_rank_pattern",
{"r": 8, "rank_pattern": {"q_proj": 32}},
"LoRA rank 32 is greater than max_lora_rank",
),
("test_rank_zero", {"r": 0}, "must be a positive integer"),
("test_rank_negative", {"r": -8}, "must be a positive integer"),
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
(
"test_pissa_niter",
{"init_lora_weights": "pissa_niter_4"},
"init_lora_weights='pissa_niter_4'",
),
("test_olora", {"init_lora_weights": "olora"}, "init_lora_weights='olora'"),
("test_corda", {"init_lora_weights": "corda"}, "init_lora_weights='corda'"),
("test_loftq", {"init_lora_weights": "loftq"}, "init_lora_weights='loftq'"),
(
"test_unknown_init",
{"init_lora_weights": "future_init"},
"init_lora_weights='future_init'",
),
("test_alora", {"alora_invocation_tokens": [1, 2]}, "Activated LoRA"),
("test_layer_replication", {"layer_replication": [[0, 2]]}, "layer_replication"),
("test_bdlora", {"use_bdlora": {"nblocks": 2}}, "BD-LoRA"),
("test_qalora", {"use_qalora": True}, "QALoRA"),
]
def test_peft_helper_pass(llama32_lora_files, tmp_path):
peft_helper = PEFTHelper.from_local_dir(
llama32_lora_files, max_position_embeddings=4096
)
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
peft_helper.validate_legal(lora_config)
assert peft_helper.r == 8
assert peft_helper.lora_alpha == 32
target_modules = sorted(peft_helper.target_modules)
assert target_modules == [
"down_proj",
"embed_tokens",
"gate_proj",
"k_proj",
"lm_head",
"o_proj",
"q_proj",
"up_proj",
"v_proj",
]
assert peft_helper.vllm_max_position_embeddings == 4096
# test RSLoRA
rslora_config = dict(use_rslora=True)
test_dir = tmp_path / "test_rslora"
shutil.copytree(llama32_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(rslora_config)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
peft_helper = PEFTHelper.from_local_dir(test_dir, max_position_embeddings=4096)
peft_helper.validate_legal(lora_config)
scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
def test_peft_helper_error(
llama32_lora_files,
tmp_path,
test_name: str,
config_change: dict,
expected_error: str,
):
test_dir = tmp_path / test_name
shutil.copytree(llama32_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(config_change)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
# Test loading the adapter
with pytest.raises(ValueError, match=expected_error):
PEFTHelper.from_local_dir(
test_dir, max_position_embeddings=4096
).validate_legal(lora_config)
@pytest.mark.parametrize("bad_rank", [0, -1, -8])
def test_peft_helper_invalid_rank_direct(bad_rank: int):
"""Regression test: constructing a PEFTHelper with a non-positive rank
must raise a clear ValueError instead of crashing with an unrelated
ZeroDivisionError (r=0) or silently succeeding with a sign-flipped
scaling factor that validate_legal() never catches (r<0, since its only
rank check is the upper bound against max_lora_rank).
Network-free: constructs PEFTHelper directly rather than going through
from_local_dir(), which needs an on-disk adapter_config.json.
"""
with pytest.raises(ValueError, match="must be a positive integer"):
PEFTHelper(r=bad_rank, lora_alpha=16, target_modules=["q_proj"])
@pytest.mark.parametrize(
"init_lora_weights",
[True, False, "gaussian", "eva", "orthogonal", "mica", "lora_ga"],
)
def test_peft_helper_init_lora_weights_supported(init_lora_weights):
"""A saved adapter with these inits loads in PEFT as a plain LoRA."""
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
PEFTHelper.from_dict(
{
"r": 8,
"lora_alpha": 16,
"target_modules": ["q_proj"],
"init_lora_weights": init_lora_weights,
"alora_invocation_tokens": None,
"layer_replication": None,
"use_bdlora": None,
}
).validate_legal(lora_config)
@pytest.mark.parametrize(
"config,module_name,expected",
[
# rank_pattern only: alpha / r_m
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.q_proj", (4, 8.0)),
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.v_proj", (16, 2.0)),
# alpha_pattern only: alpha_m / r
(
{"alpha_pattern": {"v_proj": 64}},
"model.layers.0.self_attn.v_proj",
(16, 4.0),
),
# keys only match at a "." boundary, like in PEFT
({"rank_pattern": {"proj": 4}}, "model.layers.0.self_attn.q_proj", (16, 2.0)),
# fused MoE experts from PEFT `target_parameters`: gate_up_proj is stored
# as `experts.base_layer`, down_proj as `experts`
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts.base_layer",
(2, 16.0),
),
(
{"rank_pattern": {"experts.down_proj": 2}},
"model.layers.0.mlp.experts",
(2, 16.0),
),
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts",
(16, 2.0),
),
],
)
def test_peft_helper_rank_and_alpha_pattern(config, module_name, expected):
peft_helper = PEFTHelper(
r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], **config
)
rank, scaling = peft_helper.get_rank_and_scaling(module_name)
assert rank == expected[0]
assert scaling == pytest.approx(expected[1])
def test_peft_helper_pattern_rslora():
peft_helper = PEFTHelper(
r=16,
lora_alpha=32,
target_modules=["q_proj"],
use_rslora=True,
rank_pattern={"q_proj": 4},
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (4, 32 / math.sqrt(4))
def test_peft_helper_dynamic_rank_conversion():
"""PEFT's `save_as_lora` with a dynamic rank writes r=1, lora_alpha=1 and the
same per-module values in rank_pattern and alpha_pattern, so every module
keeps a scaling of 1."""
peft_helper = PEFTHelper(
r=1,
lora_alpha=1,
target_modules=["q_proj", "v_proj"],
rank_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
alpha_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.q_proj") == (
32,
1.0,
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.v_proj") == (
3,
1.0,
)
with pytest.raises(ValueError, match="LoRA rank 32 is greater than max_lora_rank"):
peft_helper.validate_legal(LoRAConfig(max_lora_rank=16))
def test_peft_helper_null_patterns():
peft_helper = PEFTHelper.from_dict(
{
"r": 8,
"lora_alpha": 16,
"target_modules": ["q_proj"],
"rank_pattern": None,
"alpha_pattern": None,
}
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (8, 2.0)
@pytest.mark.parametrize("bad_rank", [0, -4, 2.5])
def test_peft_helper_invalid_rank_pattern(bad_rank):
with pytest.raises(ValueError, match="rank_pattern"):
PEFTHelper(
r=8,
lora_alpha=16,
target_modules=["q_proj"],
rank_pattern={"q_proj": bad_rank},
)
-75
View File
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
class DummyLoRAResolver(LoRAResolver):
"""A dummy LoRA resolver for testing."""
async def resolve_lora(
self, base_model_name: str, lora_name: str
) -> LoRARequest | None:
if lora_name == "test_lora":
return LoRARequest(
lora_name=lora_name,
lora_path=f"/dummy/path/{base_model_name}/{lora_name}",
lora_int_id=abs(hash(lora_name)),
)
return None
def test_resolver_registry_registration():
"""Test basic resolver registration functionality."""
registry = LoRAResolverRegistry
resolver = DummyLoRAResolver()
# Register a new resolver
registry.register_resolver("dummy", resolver)
assert "dummy" in registry.get_supported_resolvers()
# Get registered resolver
retrieved_resolver = registry.get_resolver("dummy")
assert retrieved_resolver is resolver
def test_resolver_registry_duplicate_registration():
"""Test registering a resolver with an existing name."""
registry = LoRAResolverRegistry
resolver1 = DummyLoRAResolver()
resolver2 = DummyLoRAResolver()
registry.register_resolver("dummy", resolver1)
registry.register_resolver("dummy", resolver2)
assert registry.get_resolver("dummy") is resolver2
def test_resolver_registry_unknown_resolver():
"""Test getting a non-existent resolver."""
registry = LoRAResolverRegistry
with pytest.raises(KeyError, match="not found"):
registry.get_resolver("unknown_resolver")
@pytest.mark.asyncio
async def test_dummy_resolver_resolve():
"""Test the dummy resolver's resolve functionality."""
dummy_resolver = DummyLoRAResolver()
base_model_name = "base_model_test"
lora_name = "test_lora"
# Test successful resolution
result = await dummy_resolver.resolve_lora(base_model_name, lora_name)
assert isinstance(result, LoRARequest)
assert result.lora_name == lora_name
assert result.lora_path == f"/dummy/path/{base_model_name}/{lora_name}"
# Test failed resolution
result = await dummy_resolver.resolve_lora(base_model_name, "nonexistent_lora")
assert result is None
-203
View File
@@ -1,203 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict
from typing import NamedTuple
from unittest.mock import MagicMock, patch
import pytest
from huggingface_hub.utils import HfHubHTTPError
from torch import nn
from vllm.lora.utils import (
get_adapter_absolute_path,
parse_fine_tuned_lora_name,
replace_submodule,
)
from vllm.model_executor.models.utils import WeightsMapper
class LoRANameParserTestConfig(NamedTuple):
name: str
module_name: str
is_lora_a: bool
weights_mapper: WeightsMapper | None = None
def test_parse_fine_tuned_lora_name_valid():
fixture = [
LoRANameParserTestConfig(
"base_model.model.lm_head.lora_A.weight", "lm_head", True, False
),
LoRANameParserTestConfig(
"base_model.model.lm_head.lora_B.weight", "lm_head", False, False
),
LoRANameParserTestConfig(
"base_model.model.model.embed_tokens.lora_embedding_A",
"model.embed_tokens",
True,
),
LoRANameParserTestConfig(
"base_model.model.model.embed_tokens.lora_embedding_B",
"model.embed_tokens",
False,
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
"model.layers.9.mlp.down_proj",
True,
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
"model.layers.9.mlp.down_proj",
False,
),
LoRANameParserTestConfig(
"language_model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.layers.9.mlp.down_proj",
True,
),
LoRANameParserTestConfig(
"language_model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.layers.9.mlp.down_proj",
False,
),
# Test with WeightsMapper
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.model.layers.9.mlp.down_proj",
True,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.model.layers.9.mlp.down_proj",
False,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.model.layers.9.mlp.down_proj",
True,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.model.layers.9.mlp.down_proj",
False,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
]
for name, module_name, is_lora_a, weights_mapper in fixture:
assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
name, weights_mapper
)
def test_parse_fine_tuned_lora_name_invalid():
fixture = {
"base_model.weight",
"base_model.model.weight",
"base_model.model.model.layers.9.mlp.down_proj.lora_A.bias",
"base_model.model.model.layers.9.mlp.down_proj.lora_B.bias",
}
for name in fixture:
with pytest.raises(ValueError, match="unsupported LoRA weight"):
parse_fine_tuned_lora_name(name)
def test_replace_submodule():
model = nn.Sequential(
OrderedDict(
[
("dense1", nn.Linear(764, 100)),
("act1", nn.ReLU()),
("dense2", nn.Linear(100, 50)),
(
"seq1",
nn.Sequential(
OrderedDict(
[
("dense1", nn.Linear(100, 10)),
("dense2", nn.Linear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("output", nn.Linear(50, 10)),
("outact", nn.Sigmoid()),
]
)
)
sigmoid = nn.Sigmoid()
replace_submodule(model, "act1", sigmoid)
assert dict(model.named_modules())["act1"] == sigmoid
dense2 = nn.Linear(1, 5)
replace_submodule(model, "seq1.dense2", dense2)
assert dict(model.named_modules())["seq1.dense2"] == dense2
# Unit tests for get_adapter_absolute_path
@patch("os.path.isabs")
def test_get_adapter_absolute_path_absolute(mock_isabs):
path = "/absolute/path/to/lora"
mock_isabs.return_value = True
assert get_adapter_absolute_path(path) == path
@patch("os.path.expanduser")
def test_get_adapter_absolute_path_expanduser(mock_expanduser):
# Path with ~ that needs to be expanded
path = "~/relative/path/to/lora"
absolute_path = "/home/user/relative/path/to/lora"
mock_expanduser.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("os.path.exists")
@patch("os.path.abspath")
def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
# Relative path that exists locally
path = "relative/path/to/lora"
absolute_path = "/absolute/path/to/lora"
mock_exist.return_value = True
mock_abspath.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("huggingface_hub.HfApi.snapshot_download")
@patch("os.path.exists")
def test_get_adapter_absolute_path_huggingface(mock_exist, mock_snapshot_download):
# Hugging Face model identifier
path = "org/repo"
absolute_path = "/mock/snapshot/path"
mock_exist.return_value = False
mock_snapshot_download.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("huggingface_hub.HfApi.snapshot_download")
@patch("os.path.exists")
def test_get_adapter_absolute_path_huggingface_error(
mock_exist, mock_snapshot_download
):
# Hugging Face model identifier with download error
path = "org/repo"
mock_exist.return_value = False
mock_snapshot_download.side_effect = HfHubHTTPError(
"failed to query model info",
response=MagicMock(),
)
assert get_adapter_absolute_path(path) == path
@@ -19,7 +19,6 @@ def test_qwen3_5_lm_head_receives_quant_config():
mock_vllm_config.cache_config.mamba_cache_mode = "align"
mock_vllm_config.scheduler_config = Mock()
mock_vllm_config.quant_config = mock_quant_config
mock_vllm_config.lora_config = None
mock_pp_group = Mock()
mock_pp_group.is_last_rank = True
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,6 @@ from transformers import AutoModelForSeq2SeqLM
from vllm.assets.audio import AudioAsset
from vllm.envs import disable_envs_cache
from vllm.lora.request import LoRARequest
from vllm.multimodal.audio import AudioResampler
from ....conftest import HfRunner, VllmRunner
@@ -26,7 +25,6 @@ AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
@@ -90,7 +88,6 @@ def test_transformers_audio_generation(
model_id: str,
):
settings = AUDIO_MODEL_SETTINGS[model_id]
audio_lora_path = settings.get("audio_lora_path")
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
target_sr = settings.get("sampling_rate", orig_sr)
@@ -105,19 +102,13 @@ def test_transformers_audio_generation(
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
enable_lora=audio_lora_path is not None,
max_lora_rank=64,
**settings.get("vllm_runner_kwargs", {}),
) as vllm_model:
lora_request = (
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
)
vllm_outputs = vllm_model.generate_greedy_logprobs(
[settings["prompt"]],
128,
num_logprobs=10,
audios=[audio],
lora_request=lora_request,
)
with hf_runner(
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from setuptools import setup
setup(
name="vllm_add_dummy_endpoint_plugin",
version="0.1",
packages=["vllm_add_dummy_endpoint_plugin"],
entry_points={
"vllm.endpoint_plugins": [
"dummy_admin_endpoint_plugin = vllm_add_dummy_endpoint_plugin:DummyAdminEndpointPlugin" # noqa
]
},
)
@@ -1,26 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worked example `vllm.endpoint_plugins` entry point.
Reports scheduler config via `collective_rpc`. Demonstrates the full
contract: `attach_router` registers the route at Phase A (`build_app`) and
`init_state` stashes the `EngineClient` the route handler needs at Phase B
(`init_app_state`).
"""
from fastapi import FastAPI, Request
class DummyAdminEndpointPlugin:
name = "dummy_admin_endpoint_plugin"
required_tasks: tuple[str, ...] | None = None
def attach_router(self, app: FastAPI) -> None:
@app.get("/v1/admin/scheduler_config")
async def scheduler_config(raw_request: Request):
engine_client = raw_request.app.state.dummy_engine_client
results = await engine_client.collective_rpc("get_scheduler_config")
return {"scheduler_config": results}
async def init_state(self, engine_client, state, args) -> None:
state.dummy_engine_client = engine_client
View File
@@ -1,130 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for GGUF plugin functionality."""
import os
from typing import NamedTuple
import pytest
from transformers import AutoTokenizer
from ...conftest import VllmRunner
from ...models.utils import check_logprobs_close
from ...utils import multi_gpu_test
os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
class GGUFTestConfig(NamedTuple):
original_model: str
gguf_model_path: str # Full path to .gguf file
QWEN3_CONFIG = GGUFTestConfig(
original_model="Qwen/Qwen3-0.6B",
gguf_model_path="unsloth/Qwen3-0.6B-GGUF:Q8_0",
)
OLMOE_CONFIG = GGUFTestConfig(
original_model="allenai/OLMoE-1B-7B-0125",
gguf_model_path="allenai/OLMoE-1B-7B-0125-GGUF:Q6_K",
)
MODELS = [
QWEN3_CONFIG,
OLMOE_CONFIG,
]
def check_model_outputs(
vllm_runner: type[VllmRunner],
prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
):
tokenizer = AutoTokenizer.from_pretrained(model.original_model)
if tokenizer.chat_template is not None:
messages = [[{"role": "user", "content": prompt}] for prompt in prompts]
prompts = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
# Run gguf model.
with vllm_runner(
model_name=model.gguf_model_path,
enforce_eager=True,
tokenizer_name=model.original_model,
dtype=dtype,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tp_size,
) as gguf_model:
gguf_outputs = gguf_model.generate_greedy_logprobs(
prompts[:-1], max_tokens, num_logprobs
)
# Run unquantized model.
# Should run with tp=1, otherwise the test will stuck at
# nccl initialization.
with vllm_runner(
model_name=model.original_model,
enforce_eager=True, # faster tests
dtype=dtype,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=1,
) as original_model:
original_outputs = original_model.generate_greedy_logprobs(
prompts[:-1], max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=original_outputs,
outputs_1_lst=gguf_outputs,
name_0="original",
name_1="gguf",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [8])
@pytest.mark.parametrize("tp_size", [1])
def test_models(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
) -> None:
check_model_outputs(
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [8])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tp_size", [2])
@multi_gpu_test(num_gpus=2)
def test_distributed(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
) -> None:
check_model_outputs(
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
)
@@ -1,167 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
os.environ["TOKENIZERS_PARALLELISM"] = "true"
from typing import Any, NamedTuple
import pytest
from pytest import MarkDecorator
from transformers import AutoModelForImageTextToText
from vllm.assets.image import ImageAsset
from vllm.multimodal.image import rescale_image_size
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import set_default_torch_num_threads
from ...conftest import IMAGE_ASSETS, HfRunner, VllmRunner
from ...models.utils import check_logprobs_close
class GGUFMMTestConfig(NamedTuple):
original_model: str
gguf_repo: str
gguf_backbone: str
gguf_mmproj: str
prompt: list[str]
image_names: list[str] # Store names, load PIL images at runtime
max_model_len: int = 4096
marks: list[MarkDecorator] = []
mm_processor_kwargs: dict[str, Any] = {}
@property
def gguf_model(self):
hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_mmproj)
return hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_backbone)
# Common prompts aligned with test_common.py "gemma3" entry format
_GEMMA3_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": (
"<bos><start_of_turn>user\n"
"<start_of_image>What's the content in the center of the image?"
"<end_of_turn>\n<start_of_turn>model\n"
),
"cherry_blossom": (
"<bos><start_of_turn>user\n"
"<start_of_image>What is the season?"
"<end_of_turn>\n<start_of_turn>model\n"
),
}
)
# Image asset names - load at runtime to avoid pickle issues with subprocess
_GEMMA3_IMAGE_NAMES = ["stop_sign", "cherry_blossom"]
# Regular multimodal (no pan-and-scan) - uses QAT Q4_0 GGUF
GEMMA3_CONFIG = GGUFMMTestConfig(
original_model="google/gemma-3-4b-it",
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
gguf_mmproj="mmproj-model-f16-4B.gguf",
prompt=_GEMMA3_PROMPTS,
image_names=_GEMMA3_IMAGE_NAMES,
max_model_len=4096,
mm_processor_kwargs={},
)
# Pan-and-scan multimodal - uses unquantized BF16 GGUF
GEMMA3_CONFIG_PAN_AND_SCAN = GGUFMMTestConfig(
original_model="google/gemma-3-4b-it",
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
gguf_mmproj="mmproj-model-f16-4B.gguf",
prompt=_GEMMA3_PROMPTS,
image_names=_GEMMA3_IMAGE_NAMES,
max_model_len=4096,
mm_processor_kwargs={"do_pan_and_scan": True},
)
MODELS_TO_TEST = [GEMMA3_CONFIG, GEMMA3_CONFIG_PAN_AND_SCAN]
def run_multimodal_gguf_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: GGUFMMTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
):
# Load images at runtime (inside subprocess) to avoid pickle issues
images = [ImageAsset(name).pil_image for name in model.image_names]
size_factors = [0.25, 0.5, 1.0]
inputs_per_image = [
(
[prompt for _ in size_factors],
[rescale_image_size(image, factor) for factor in size_factors],
)
for image, prompt in zip(images, model.prompt)
]
# NOTE: Run vLLM first to avoid CUDA init issues with multiprocessing fork.
# Run GGUF model via vLLM.
with (
set_default_torch_num_threads(1),
vllm_runner(
model_name=model.gguf_model,
enforce_eager=True,
tokenizer_name=model.original_model,
dtype=dtype,
max_model_len=model.max_model_len,
mm_processor_kwargs=model.mm_processor_kwargs,
) as gguf_model,
):
gguf_outputs_per_case = [
gguf_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
)
for prompts, images in inputs_per_image
]
# Then run HfRunner for HuggingFace baseline comparison.
with hf_runner(
model.original_model,
dtype=dtype,
auto_cls=AutoModelForImageTextToText,
) as hf_model:
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
)
for prompts, images in inputs_per_image
]
for hf_outputs, gguf_outputs in zip(hf_outputs_per_case, gguf_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=gguf_outputs,
name_0="hf",
name_1="gguf",
)
@pytest.mark.parametrize("model", MODELS_TO_TEST)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [10])
def test_gemma3_mm_gguf(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: GGUFMMTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
run_multimodal_gguf_test(
hf_runner, vllm_runner, model, dtype, max_tokens, num_logprobs
)
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import shutil
import pytest
from vllm.plugins.lora_resolvers.filesystem_resolver import FilesystemResolver
from vllm.transformers_utils.repo_utils import hf_api
MODEL_NAME = "Qwen/Qwen3-0.6B"
LORA_NAME = "charent/self_cognition_Alice"
PA_NAME = "swapnilbp/llama_tweet_ptune"
@pytest.fixture(scope="module")
def adapter_cache(request, tmpdir_factory):
# Create dir that mimics the structure of the adapter cache
adapter_cache = tmpdir_factory.mktemp(request.module.__name__) / "adapter_cache"
return adapter_cache
@pytest.fixture(scope="module")
def qwen3_lora_files():
return hf_api().snapshot_download(repo_id=LORA_NAME)
@pytest.fixture(scope="module")
def pa_files():
return hf_api().snapshot_download(repo_id=PA_NAME)
@pytest.mark.asyncio
async def test_filesystem_resolver(adapter_cache, qwen3_lora_files):
model_files = adapter_cache / LORA_NAME
shutil.copytree(qwen3_lora_files, model_files)
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
lora_request = await fs_resolver.resolve_lora(MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert lora_request.lora_path == os.path.join(adapter_cache, LORA_NAME)
@pytest.mark.asyncio
async def test_missing_adapter(adapter_cache):
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
missing_lora_request = await fs_resolver.resolve_lora(MODEL_NAME, "foobar")
assert missing_lora_request is None
@pytest.mark.asyncio
async def test_nonlora_adapter(adapter_cache, pa_files):
model_files = adapter_cache / PA_NAME
shutil.copytree(pa_files, model_files)
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
pa_request = await fs_resolver.resolve_lora(MODEL_NAME, PA_NAME)
assert pa_request is None
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from huggingface_hub.constants import HF_HUB_CACHE
from vllm.plugins.lora_resolvers.hf_hub_resolver import HfHubResolver
LORA_LIB_MODEL_NAME = "ibm-granite/granite-3.3-8b-instruct"
# Repo with multiple LoRAs contained in it
LORA_LIB = "ibm-granite/granite-3.3-8b-rag-agent-lib"
LORA_NAME = "ibm-granite/granite-3.3-8b-rag-agent-lib/answerability_prediction_lora" # noqa: E501
NON_LORA_SUBPATH = "ibm-granite/granite-3.3-8b-rag-agent-lib/README.md"
LIB_DOWNLOAD_DIR = os.path.join(
HF_HUB_CACHE, "models--ibm-granite--granite-3.3-8b-rag-agent-lib"
)
INVALID_REPO_NAME = "thisrepodoesnotexist"
# Repo with only one LoRA in the root dir
LORA_REPO_MODEL_NAME = "meta-llama/Llama-2-7b-hf"
LORA_REPO = "yard1/llama-2-7b-sql-lora-test"
REPO_DOWNLOAD_DIR = os.path.join(
HF_HUB_CACHE, "models--yard1--llama-2-7b-sql-lora-test"
)
@pytest.mark.asyncio
async def test_hf_resolver_with_direct_path():
hf_resolver = HfHubResolver([LORA_REPO])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_REPO_MODEL_NAME, LORA_REPO)
assert lora_request is not None
assert lora_request.lora_name == LORA_REPO
assert REPO_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_hf_resolver_with_nested_paths():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_hf_resolver_with_multiple_repos():
hf_resolver = HfHubResolver([LORA_LIB, LORA_REPO])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_missing_adapter():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
missing_lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, "foobar")
assert missing_lora_request is None
@pytest.mark.asyncio
async def test_nonlora_adapter():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
readme_request = await hf_resolver.resolve_lora(
LORA_LIB_MODEL_NAME, NON_LORA_SUBPATH
)
assert readme_request is None
@pytest.mark.asyncio
async def test_invalid_repo():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
invalid_repo_req = await hf_resolver.resolve_lora(
INVALID_REPO_NAME,
f"{INVALID_REPO_NAME}/foo",
)
assert invalid_repo_req is None
@pytest.mark.asyncio
async def test_trailing_slash():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(
LORA_LIB_MODEL_NAME,
f"{LORA_NAME}/",
)
assert lora_request is not None
assert lora_request.lora_name == f"{LORA_NAME}/"
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@@ -1,363 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the `vllm.endpoint_plugins` framework (RFC #46565).
Uses the worked in repo example plugin (`vllm_add_dummy_endpoint_plugin`,
installed via `tests/plugins/vllm_add_dummy_endpoint_plugin`) exercising both
`EndpointPlugin` hooks against a fake `EngineClient`, unit tests for the
`load_endpoint_plugins` gating matrix and an e2e test that drives a real HTTP
request through the plugin's route.
"""
from argparse import Namespace
from typing import Any
import httpx
import pytest
from fastapi import APIRouter, FastAPI
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
from vllm.entrypoints.launchers.app import build_app
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.plugins import load_endpoint_plugins
from vllm.plugins.endpoint_plugins.interface import (
EndpointPlugin,
attach_endpoint_plugins,
init_endpoint_plugins_state,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
class _RaisingEndpointPlugin:
"""Factory that raises to exercise the "instantiation fails" path."""
name = "raising_endpoint_plugin"
required_tasks = None
def __init__(self):
raise RuntimeError("boom")
class _ShadowEndpointPlugin:
name = "shadow_endpoint_plugin"
required_tasks = None
def __init__(self, path: str, include_in_schema: bool):
self.path = path
self.include_in_schema = include_in_schema
def attach_router(self, app: FastAPI) -> None:
assert str(app.url_path_for("show_version")) == "/version"
app.openapi()
@app.get(self.path, status_code=209, include_in_schema=self.include_in_schema)
async def plugin_version(plugin_arg: str) -> dict[str, str]:
return {"served_by": plugin_arg}
async def init_state(self, engine_client, state, args) -> None:
pass
class _RouterEndpointPlugin(DummyAdminEndpointPlugin):
def __init__(self, router: APIRouter):
self.router = router
def attach_router(self, app: FastAPI) -> None:
app.include_router(self.router)
class _FakeEngineClient:
"""Minimal stand in exercising `collective_rpc`. Not a real engine."""
def __init__(self, rpc_result: Any = None):
self.rpc_result = rpc_result
self.rpc_calls: list[tuple[str, tuple, dict]] = []
async def collective_rpc(self, method, timeout=None, args=(), kwargs=None):
self.rpc_calls.append((method, args, kwargs or {}))
return self.rpc_result
def _build_args() -> Namespace:
parser = FlexibleArgumentParser()
subparsers = parser.add_subparsers()
serve_parser = subparsers.add_parser("serve")
make_arg_parser(serve_parser)
return serve_parser.parse_args([])
def _fake_loader(factories: dict[str, Any]):
def _load_plugins_by_group(group: str) -> dict[str, Any]:
assert group == "vllm.endpoint_plugins"
return factories
return _load_plugins_by_group
def test_dummy_plugin_satisfies_protocol():
assert isinstance(DummyAdminEndpointPlugin(), EndpointPlugin)
def test_no_plugins_loaded_when_allowlist_unset(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
assert load_endpoint_plugins(("generate",)) == []
def test_no_plugins_loaded_when_allowlist_is_empty_string(
monkeypatch: pytest.MonkeyPatch,
):
"""`VLLM_PLUGINS=""` parses to `[""]`, not `None` (see `vllm.envs`), so it
must be treated as a (non strict) allowlist matching no plugin name, not
as "unset"."""
monkeypatch.setenv("VLLM_PLUGINS", "")
assert load_endpoint_plugins(("generate",)) == []
def test_plugin_loaded_when_allowlisted_and_task_matches(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
plugins = load_endpoint_plugins(("generate",))
assert len(plugins) == 1
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
def test_plugin_skipped_when_required_tasks_miss(monkeypatch: pytest.MonkeyPatch):
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
required_tasks = ("generate",)
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
)
assert load_endpoint_plugins(("embed",)) == []
assert len(load_endpoint_plugins(("generate",))) == 1
def test_plugin_loaded_when_required_tasks_is_none(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
assert len(load_endpoint_plugins(supported_tasks=None)) == 1
def test_plugin_skipped_when_required_tasks_set_but_supported_tasks_none(
monkeypatch: pytest.MonkeyPatch,
):
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
required_tasks = ("generate",)
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
)
assert load_endpoint_plugins(supported_tasks=None) == []
def test_factory_raising_is_logged_and_skipped(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(
"VLLM_PLUGINS", "raising_endpoint_plugin,dummy_admin_endpoint_plugin"
)
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader(
{
"raising_endpoint_plugin": _RaisingEndpointPlugin,
"dummy_admin_endpoint_plugin": DummyAdminEndpointPlugin,
}
),
)
plugins = load_endpoint_plugins(("generate",))
assert len(plugins) == 1
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
app = FastAPI()
attach_endpoint_plugins(app, ("generate",))
assert app.state.endpoint_plugins == []
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["/version", "/metrics"])
@pytest.mark.parametrize("include_in_schema", [True, False])
async def test_endpoint_plugin_override_preserves_hook_and_schema(
monkeypatch: pytest.MonkeyPatch, path: str, include_in_schema: bool
):
"""Hooks see core routes; overrides win over both HTTP routes and mounts."""
args = _build_args()
monkeypatch.setenv("VLLM_PLUGINS", "shadow_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader(
{
"shadow_endpoint_plugin": lambda: _ShadowEndpointPlugin(
path, include_in_schema
)
}
),
)
app = build_app(args, supported_tasks=())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(path, params={"plugin_arg": "plugin"})
schema = (await client.get("/openapi.json")).json()
assert response.status_code == 209
assert response.json() == {"served_by": "plugin"}
if include_in_schema:
operation = schema["paths"][path]["get"]
assert "209" in operation["responses"]
assert "200" not in operation["responses"]
assert operation["parameters"][0]["name"] == "plugin_arg"
assert operation["parameters"][0]["required"] is True
response_schema = operation["responses"]["209"]["content"]["application/json"][
"schema"
]
assert response_schema["additionalProperties"] == {"type": "string"}
else:
assert path not in schema["paths"]
@pytest.mark.asyncio
async def test_endpoint_plugin_override_preserves_other_methods(
monkeypatch: pytest.MonkeyPatch,
):
"""Overriding GET preserves POST's validation and the shared core route."""
core = APIRouter()
@core.api_route("/shared", methods=["GET", "POST"], status_code=201)
async def original(value: int) -> dict[str, int]:
return {"value": value}
plugin = APIRouter()
@plugin.get("/shared", status_code=202)
async def replacement():
return {"served_by": "plugin"}
app = FastAPI()
app.include_router(core)
original_route = app.routes[-1]
monkeypatch.setattr(
"vllm.plugins.load_endpoint_plugins",
lambda supported_tasks: [_RouterEndpointPlugin(plugin)],
)
attach_endpoint_plugins(app, ())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/shared")
post_response = await client.post("/shared", params={"value": 7})
invalid_response = await client.post("/shared", params={"value": "invalid"})
assert response.status_code == 202
assert response.json() == {"served_by": "plugin"}
assert post_response.status_code == 201
assert post_response.json() == {"value": 7}
assert invalid_response.status_code == 422
assert original_route.methods == {"GET", "POST"}
assert core.routes[0].methods == {"GET", "POST"}
operations = app.openapi()["paths"]["/shared"]
assert "202" in operations["get"]["responses"]
assert "201" in operations["post"]["responses"]
@pytest.mark.asyncio
async def test_later_endpoint_plugin_overrides_same_operation(
monkeypatch: pytest.MonkeyPatch,
):
"""The last registered override wins while unrelated plugin routes survive."""
first = APIRouter()
@first.get("/shared", status_code=201)
@first.get("/plugins/first")
async def first_handler():
return {"served_by": "first"}
second = APIRouter()
@second.get("/shared", status_code=202)
async def second_handler():
return {"served_by": "second"}
monkeypatch.setattr(
"vllm.plugins.load_endpoint_plugins",
lambda supported_tasks: [
_RouterEndpointPlugin(first),
_RouterEndpointPlugin(second),
],
)
app = FastAPI()
attach_endpoint_plugins(app, ())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/shared")
unrelated = await client.get("/plugins/first")
unsupported = await client.delete("/shared")
assert response.status_code == 202
assert response.json() == {"served_by": "second"}
assert unrelated.status_code == 200
assert unrelated.json() == {"served_by": "first"}
assert unsupported.status_code == 405
responses = app.openapi()["paths"]["/shared"]["get"]["responses"]
assert "202" in responses
assert "201" not in responses
@pytest.mark.asyncio
async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatch):
"""A caller that builds a bare `State()` without `build_app` must not crash
just because `state.endpoint_plugins` was never set."""
from starlette.datastructures import State
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
state = State()
await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
assert not hasattr(state, "dummy_engine_client")
@pytest.mark.asyncio
async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):
"""Phase A (attach) + Phase B (init) wired through `build_app` then
exercised with a real HTTP request against the worked example plugin."""
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
args = _build_args()
app = build_app(args, supported_tasks=())
assert len(app.state.endpoint_plugins) == 1
assert any(
getattr(route, "path", None) == "/v1/admin/scheduler_config"
for route in app.routes
)
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
await init_endpoint_plugins_state(fake_engine_client, app.state, args)
assert app.state.dummy_engine_client is fake_engine_client
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/v1/admin/scheduler_config")
assert response.status_code == 200
assert response.json() == {"scheduler_config": ["cfg-a", "cfg-b"]}
assert fake_engine_client.rpc_calls == [("get_scheduler_config", (), {})]
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.utils import VLLM_PATH, RemoteOpenAIServer
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
assert chatml_jinja_path.exists()
def run_and_test_dummy_opt_api_server(model, tp=1, extra_args=(), env_dict=None):
# the model is registered through the plugin
server_args = [
"--gpu-memory-utilization",
"0.10",
"--dtype",
"float32",
"--chat-template",
str(chatml_jinja_path),
"--load-format",
"dummy",
"-tp",
f"{tp}",
*extra_args,
]
with RemoteOpenAIServer(model, server_args, env_dict=env_dict) as server:
client = server.get_client()
completion = client.chat.completions.create(
model=model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello!"},
],
temperature=0,
)
generated_text = completion.choices[0].message.content
assert generated_text is not None
# make sure only the first token is generated
rest = generated_text.replace("<s>", "")
assert rest == ""
def test_oot_registration_for_api_server(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path)
-1
View File
@@ -1049,7 +1049,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
config = SimpleNamespace(
speculative_config=SimpleNamespace(enable_adaptive_verification=True),
lora_config=None,
compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
)
-1
View File
@@ -22,7 +22,6 @@ def _create_new_requests_data(prompt_embeds: torch.Tensor | None) -> NewRequestD
pooling_params=None,
block_ids=([],),
num_computed_tokens=0,
lora_request=None,
prompt_embeds=prompt_embeds,
)
-65
View File
@@ -19,7 +19,6 @@ from vllm.distributed.kv_events import (
BlockRemoved,
BlockStored,
)
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
@@ -74,7 +73,6 @@ def make_request(
mm_hashes: list[str] | None = None,
prompt_logprobs: int | None = None,
cache_salt: str | None = None,
lora_request: LoRARequest | None = None,
session_id: str | None = None,
):
mm_features = []
@@ -98,7 +96,6 @@ def make_request(
mm_features=mm_features if mm_features else None,
sampling_params=sampling_params,
pooling_params=None,
lora_request=lora_request,
cache_salt=cache_salt,
block_hasher=get_request_block_hasher(block_size, hash_fn),
session_id=session_id,
@@ -2356,64 +2353,6 @@ def test_null_parent_block_hash():
assert blocks[num_full_blocks - 1].block_hash is not None
@pytest.mark.parametrize("blocks_to_cache", [2, 3, 10])
def test_kv_cache_events_with_lora(blocks_to_cache: int):
"""Test BlockStored events contain correct lora_id when using LoRA requests."""
block_size = 16
num_blocks = blocks_to_cache + 1
# Create KVCacheManager with events enabled
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, num_blocks),
max_model_len=8192,
enable_caching=True,
enable_kv_cache_events=True,
hash_block_size=block_size,
)
# Test with LoRA request
lora_request = LoRARequest(
lora_name="test_lora", lora_int_id=42, lora_path="/test/path"
)
num_tokens = block_size * blocks_to_cache
req_with_lora = make_request(
"lora_req",
list(range(num_tokens)),
block_size,
sha256,
lora_request=lora_request,
)
# Allocate slots and get events
_ = manager.allocate_slots(req_with_lora, num_tokens)
events = manager.take_events()
# Verify BlockStored event contains correct lora_id
block_stored_event = events[-1]
assert isinstance(block_stored_event, BlockStored)
assert block_stored_event.lora_id == 42 # Should match lora_request.adapter_id
assert len(block_stored_event.block_hashes) == blocks_to_cache
assert block_stored_event.block_size == block_size
# Clean up
manager.free(req_with_lora)
# Test without LoRA request (should have lora_id=None)
req_without_lora = make_request(
"no_lora_req", list(range(num_tokens)), block_size, sha256
)
_ = manager.allocate_slots(req_without_lora, num_tokens)
events = manager.take_events()
block_stored_event = events[-1]
assert isinstance(block_stored_event, BlockStored)
assert block_stored_event.lora_id is None # Should be None when no LoRA request
assert len(block_stored_event.block_hashes) == blocks_to_cache
assert block_stored_event.block_size == block_size
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_stored_event_group_idx(group_id: int):
"""Test BlockStored events emitted by cache_full_blocks carry the correct
@@ -2634,9 +2573,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
parent_block_hash=None,
token_ids=list(range(block_size)),
block_size=block_size,
lora_id=None,
medium=None,
lora_name=None,
group_idx=1,
)
manager.block_pool.kv_event_queue.append(event)
@@ -2764,8 +2701,6 @@ def test_emit_cached_block_events():
assert event.group_idx == kv_cache_group_id
assert event.block_size == block_size
assert event.medium == MEDIUM_GPU
assert event.lora_id is None
assert event.lora_name is None
assert event.session_id == "agent-session-reuse"
+2 -12
View File
@@ -382,9 +382,7 @@ def _make_spec_decode_manager(
def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
assert [
(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[(12, 0)]
] == [
assert [(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[12]] == [
(CUDAGraphMode.FULL, 18),
(CUDAGraphMode.PIECEWISE, 16),
]
@@ -393,7 +391,6 @@ def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
num_reqs=4,
num_tokens=12,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
@@ -409,7 +406,6 @@ def test_uniform_decode_exact_match_is_not_over_padded(monkeypatch):
num_reqs=3,
num_tokens=9,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
@@ -424,7 +420,6 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
num_reqs=2,
num_tokens=12,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
@@ -443,7 +438,6 @@ def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
@@ -479,7 +473,6 @@ def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
num_reqs=1,
num_tokens=num_tokens,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE, num_tokens
assert desc.uniform_token_count is None, num_tokens
@@ -493,7 +486,6 @@ def test_uniform_decode_beyond_capture_ladder_falls_back(monkeypatch):
num_reqs=9,
num_tokens=27,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.NONE
@@ -529,13 +521,12 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
for desc in full_descs:
assert desc.num_reqs is not None
assert desc in manager._candidates[(desc.num_tokens, 0)], desc
assert desc in manager._candidates[desc.num_tokens], desc
assert (
manager.dispatch(
num_reqs=desc.num_reqs,
num_tokens=desc.num_tokens,
uniform_token_count=desc.uniform_token_count,
num_active_loras=0,
)
== desc
), desc
@@ -633,7 +624,6 @@ def test_divisor_query_len_dispatch_is_unchanged(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=decode_query_len,
num_active_loras=0,
)
expected = min(s for s in capture_sizes if s >= num_tokens)
assert desc.cg_mode == CUDAGraphMode.FULL, num_tokens
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction."""
import pytest
import torch
from utils import skip_if_not_cuda_alike
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
import vllm.lora.ops.triton_ops.utils as lora_utils
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT
from vllm.utils.torch_utils import set_random_seed
DEVICE = "cuda"
@pytest.fixture(autouse=True)
def batch_invariant(monkeypatch):
monkeypatch.setattr(lora_utils, "is_batch_invariant", True)
monkeypatch.setattr(shrink_op, "is_batch_invariant", True)
lora_utils.get_lora_op_configs.cache_clear()
yield
lora_utils.get_lora_op_configs.cache_clear()
def _shrink(inputs, weights, mapping, scaling):
num_tokens = inputs.size(0)
meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE)
meta.prepare_tensors(mapping)
out = torch.zeros(
(len(weights), num_tokens, weights[0].size(1)),
dtype=torch.float32,
device=DEVICE,
)
_LORA_A_PTR_DICT.clear()
args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False)
lora_shrink(inputs, weights, out, *args, scaling)
return out
@skip_if_not_cuda_alike
@pytest.mark.parametrize(
"hidden_size,rank,nslices,dtype",
[
(2560, 8, 3, torch.bfloat16),
(9728, 16, 1, torch.float16),
(4097, 64, 1, torch.bfloat16), # K tail block
(1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty
],
)
def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
set_random_seed(0)
inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE)
weights = [
torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE)
out = _shrink(inputs, weights, mapping, 0.5).double().cpu()
ids, x = mapping.cpu(), inputs.double().cpu()
ref = torch.zeros_like(out)
for s, weight in enumerate(weights):
for lora_id in (0, 1):
rows = ids == lora_id
ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
@skip_if_not_cuda_alike
@pytest.mark.parametrize("nslices", [1, 3])
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
class RecordReduce:
def __getitem__(self, grid):
reduce_grids.append(grid)
return reduce_kernel[grid]
monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce())
set_random_seed(0)
# K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute.
dtype, hidden_size = torch.bfloat16, 2560
weights = [
torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE)
target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE)
expected = _shrink(target, weights, target_id, 0.7)[:, 0]
for num_tokens in (31, 32, 33, 127, 128, 129):
inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE)
mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE)
pos = num_tokens // 2
inputs[pos], mapping[pos] = target[0], 0
out = _shrink(inputs, weights, mapping, 0.7)
assert torch.equal(out[:, pos], expected), num_tokens
assert len(reduce_grids) == 7
@@ -96,7 +96,6 @@ def _attach_prometheus_logger(
vllm_config.speculative_config = None
vllm_config.kv_transfer_config = None
vllm_config.ec_transfer_config = None
vllm_config.lora_config = None
llm.logger_manager = StatLoggerManager(
vllm_config=vllm_config,
engine_idxs=engine_indexes,
@@ -128,7 +127,6 @@ def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest:
sampling_params=SamplingParams(n=n),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
@@ -47,8 +46,6 @@ def _run_dummy_batch(
speculator=speculator,
kv_connector=SimpleNamespace(set_disabled=lambda disabled: None),
is_first_pp_rank=True,
lora_config=None,
maybe_dummy_run_with_lora=lambda *args, **kwargs: nullcontext(),
execute_model=execute_model,
)
@@ -137,7 +137,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
# With DSD enabled, MRv2 should have captured a FULL candidate for
@@ -146,7 +145,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
assert desc.uniform_token_count == max_query_len
assert desc.num_tokens == num_tokens
assert desc.num_reqs == num_reqs
assert desc.num_active_loras == 0
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
@@ -216,14 +214,12 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
num_reqs=2,
num_tokens=3,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_reqs is None
assert desc.num_tokens == 3
assert desc.num_active_loras == 0
def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
@@ -276,7 +272,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
@@ -290,7 +285,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == decode_query_len
@@ -344,14 +338,12 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
@@ -409,7 +401,6 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
if max_query_len in scheduled_query_lens:
@@ -425,4 +416,3 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
@@ -35,7 +35,6 @@ def mock_model_runner_with_req_states():
runner.encoder_cache = None
runner.model_state = Mock()
runner.block_tables = Mock()
runner.lora_state = Mock()
runner.pp_handler = None
runner.sampler = None
runner.prompt_logprobs_worker = None
@@ -91,7 +90,6 @@ def test_e2e_streaming_request_update_basic_flow(
pooling_params=None,
block_ids=([0],),
num_computed_tokens=3,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([initial_req_data]))
assert req_id in req_states.req_id_to_index
@@ -109,7 +107,6 @@ def test_e2e_streaming_request_update_basic_flow(
pooling_params=None,
block_ids=([0, 1],),
num_computed_tokens=4, # 3 original prompt + 1 intermediate output
lora_request=None,
)
runner.add_requests(_make_scheduler_output([updated_req_data]))
@@ -163,7 +160,6 @@ def test_e2e_streaming_with_multimodal_features(
pooling_params=None,
block_ids=([0],),
num_computed_tokens=14,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([initial_req_data]))
assert req_id in req_states.req_id_to_index
@@ -185,7 +181,6 @@ def test_e2e_streaming_with_multimodal_features(
pooling_params=None,
block_ids=([0, 1],),
num_computed_tokens=14,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([updated_req_data]))
@@ -270,8 +270,6 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
needs_capture=lambda: captured,
capture=lambda *args, **kwargs: None,
)
runner.lora_config = None
runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext()
runner.speculator = None
runner.adaptive_verification = None
runner.model = None
@@ -363,8 +363,6 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch):
runner.kv_caches = []
runner.attn_groups = []
runner.kv_cache_config = SimpleNamespace()
runner.lora_config = None
runner.maybe_remove_all_loras = lambda _: None
allocation_bytes = 64 << 20
memory: dict[str, int] = {}
@@ -432,8 +430,6 @@ def test_teardown_profiling_state_clears_mamba_align_metadata(monkeypatch):
runner.attn_groups = []
runner.kv_cache_config = SimpleNamespace()
runner.cudagraph_manager = object()
runner.lora_config = None
runner.maybe_remove_all_loras = lambda _: None
monkeypatch.setattr(cgu.torch.accelerator, "synchronize", lambda: None)
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
@@ -81,7 +81,6 @@ def _make_runner(**overrides: Any) -> Any:
load_config=runner.load_config,
model_config=runner.model_config,
)
runner.lora_config = None
runner.use_aux_hidden_state_outputs = False
runner.speculative_config = None
runner.speculator = None
@@ -8,12 +8,9 @@ session is active. These tests verify that delegation and the session guard.
"""
import pytest
import torch
import torch.nn as nn
from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config
from vllm.lora.layers import BaseLayerWithLoRA
from vllm.model_executor.model_loader.reload.inplace import get_parameter_for_reload
from vllm.v1.worker.gpu_worker import Worker
@@ -57,13 +54,13 @@ class _RecordingEngine:
class _RecordingModelRunner:
def __init__(self) -> None:
self.seen_config: VllmConfig | None = None
self.reset_lora_calls = 0
self.refresh_draft_weights_calls = 0
def reload_weights(self) -> None:
self.seen_config = get_current_vllm_config()
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def refresh_draft_weights(self) -> None:
self.refresh_draft_weights_calls += 1
def get_draft_model(self) -> nn.Module | None:
return None
@@ -75,6 +72,7 @@ def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker.weight_transfer_engine = engine
worker._weight_update_active = False
worker._weight_update_is_draft = False
worker.use_v2_model_runner = False
worker.model_runner = _RecordingModelRunner()
return worker
@@ -108,22 +106,6 @@ def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
assert worker.model_runner is model_runner
def test_reload_parameter_lookup_preserves_lora_module_names():
base_layer = nn.Module()
qweight = nn.Parameter(torch.ones(1))
base_layer.register_parameter("qweight", qweight)
wrapper = BaseLayerWithLoRA()
wrapper.base_layer = base_layer
model = nn.Module()
model.proj = wrapper
named_parameters = dict(model.named_parameters())
assert set(named_parameters) == {"proj.base_layer.qweight"}
assert named_parameters["proj.base_layer.qweight"] is qweight
assert model.get_parameter("proj.base_layer.qweight") is qweight
assert get_parameter_for_reload(model, "proj.qweight") is qweight
def test_start_update_finish_delegates_to_engine():
engine = _RecordingEngine()
worker = _make_worker(engine)
@@ -141,7 +123,7 @@ def test_start_update_finish_delegates_to_engine():
assert engine.reset_count == 1
assert worker._weight_update_active is False
assert engine.seen_configs == [worker.vllm_config] * 3
assert worker.model_runner.reset_lora_calls == 1
assert worker.model_runner.refresh_draft_weights_calls == 0
@pytest.mark.parametrize(
@@ -194,7 +176,7 @@ def test_rank_local_update_uses_data_parallel_index_after_reconfigure():
assert worker._weight_update_active is True
def test_finish_draft_session_keeps_lora_state():
def test_finish_draft_session_does_not_refresh_draft_weights():
engine = _RecordingEngine()
engine.supports_draft_weight_update = True
worker = _make_worker(engine)
@@ -203,7 +185,7 @@ def test_finish_draft_session_keeps_lora_state():
Worker.start_draft_weight_update(worker)
Worker.finish_weight_update(worker)
assert worker.model_runner.reset_lora_calls == 0
assert worker.model_runner.refresh_draft_weights_calls == 0
def test_double_start_raises():