[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1534,29 +1534,10 @@ def num_gpus_available():
|
||||
|
||||
|
||||
temp_dir = tempfile.gettempdir()
|
||||
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
|
||||
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
|
||||
_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_opt_path():
|
||||
json_path = os.path.join(_dummy_opt_path, "config.json")
|
||||
if not os.path.exists(_dummy_opt_path):
|
||||
hf_api().snapshot_download(
|
||||
repo_id="facebook/opt-125m",
|
||||
local_dir=_dummy_opt_path,
|
||||
ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
|
||||
)
|
||||
assert os.path.exists(json_path)
|
||||
with open(json_path) as f:
|
||||
config = json.load(f)
|
||||
config["architectures"] = ["MyOPTForCausalLM"]
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(config, f)
|
||||
return _dummy_opt_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_llava_path():
|
||||
json_path = os.path.join(_dummy_llava_path, "config.json")
|
||||
|
||||
@@ -39,7 +39,6 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
|
||||
sampling_params=params,
|
||||
pooling_params=None,
|
||||
arrival_time=0.0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.plugins_tests.test_oot_registration_online import (
|
||||
run_and_test_dummy_opt_api_server,
|
||||
)
|
||||
|
||||
|
||||
def test_distributed_oot(dummy_opt_path: str):
|
||||
run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
def test_distributed_oot_pp_raw_input_tokens(dummy_opt_path: str, enforce_eager: bool):
|
||||
# The model asserts that input_ids reaches every pipeline stage.
|
||||
overrides = {"architectures": ["MyOPTRawInputTokensForCausalLM"]}
|
||||
extra_args = ["-pp", "2", "--hf-overrides", json.dumps(overrides)]
|
||||
if enforce_eager:
|
||||
extra_args.append("--enforce-eager")
|
||||
run_and_test_dummy_opt_api_server(
|
||||
dummy_opt_path,
|
||||
extra_args=extra_args,
|
||||
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
)
|
||||
@@ -24,9 +24,7 @@ class _LegacyBlockStored(
|
||||
parent_block_hash: bytes | None
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
lora_id: int | None
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
group_idx: int | None = None
|
||||
kv_cache_spec_kind: str | None = None
|
||||
@@ -58,9 +56,7 @@ class _PreSessionBlockStored(
|
||||
parent_block_hash: bytes | None
|
||||
token_ids: list[int]
|
||||
block_size: int
|
||||
lora_id: int | None
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
group_idx: int | None = None
|
||||
kv_cache_spec_kind: str | None = None
|
||||
@@ -79,9 +75,7 @@ def _make_block_stored(
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=group_idx,
|
||||
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
|
||||
locality=locality,
|
||||
@@ -183,9 +177,7 @@ def test_block_stored_locality_is_wire_compatible():
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
)
|
||||
@@ -221,9 +213,7 @@ def test_block_stored_session_id_is_wire_compatible():
|
||||
parent_block_hash=None,
|
||||
token_ids=[1, 2, 3, 4],
|
||||
block_size=4,
|
||||
lora_id=None,
|
||||
medium="GPU",
|
||||
lora_name=None,
|
||||
group_idx=2,
|
||||
kv_cache_spec_sliding_window=128,
|
||||
locality="LOCAL",
|
||||
|
||||
@@ -65,37 +65,3 @@ def sample_structured_outputs_choices():
|
||||
"Swift",
|
||||
"Kotlin",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_meowing_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_woofing_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def opt125_lora_files() -> str:
|
||||
"""Download opt-125m LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="peft-internal-testing/opt-125m-dummy-lora"
|
||||
)
|
||||
|
||||
@@ -1,28 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
LORA_MODULE = {
|
||||
"name": "module2",
|
||||
"path": "/path/to/module2",
|
||||
"base_model_name": "llama",
|
||||
}
|
||||
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
assert CHATML_JINJA_PATH.exists()
|
||||
|
||||
|
||||
def _build_vllm_parsers():
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
@@ -96,87 +84,6 @@ def test_logging_config_cli_args(serve_parser):
|
||||
assert config.pylogging_config_file == "/tmp/flat.json"
|
||||
|
||||
|
||||
### Tests for LoRA module parsing
|
||||
def test_valid_key_value_format(serve_parser):
|
||||
# Test old format: name=path
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
]
|
||||
)
|
||||
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_valid_json_format(serve_parser):
|
||||
# Test valid JSON format input
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_invalid_json_format(serve_parser):
|
||||
# Test invalid JSON format input, missing closing brace
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_type_error(serve_parser):
|
||||
# Test type error when values are not JSON or key=value
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"invalid_format", # This is not JSON or key=value format
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_json_field(serve_parser):
|
||||
# Test valid JSON format but missing required fields
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
'{"name": "module4"}', # Missing required 'path' field
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_empty_values(serve_parser):
|
||||
# Test when no LoRA modules are provided
|
||||
args = serve_parser.parse_args(["--lora-modules", ""])
|
||||
assert args.lora_modules == []
|
||||
|
||||
|
||||
def test_multiple_valid_inputs(serve_parser):
|
||||
# Test multiple valid inputs (both old and JSON format)
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module1", path="/path/to/module1"),
|
||||
LoRAModulePath(
|
||||
name="module2", path="/path/to/module2", base_model_name="llama"
|
||||
),
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
|
||||
|
||||
@@ -241,10 +148,12 @@ def test_reasoning_parser_none_disables_parser(serve_parser, value, expected):
|
||||
assert args.reasoning_parser == expected
|
||||
|
||||
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser):
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser, tmp_path):
|
||||
"""Ensure validation passes if the chat template exists."""
|
||||
chat_template = tmp_path / "template.jinja"
|
||||
chat_template.write_text("{{ messages }}")
|
||||
args = serve_parser.parse_args(
|
||||
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
|
||||
args=["--chat-template", chat_template.absolute().as_posix()]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
@@ -389,32 +298,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||
else:
|
||||
with pytest.raises(raises):
|
||||
vllm_parser.parse_args(args=args)
|
||||
|
||||
|
||||
### Tests for LoRA target modules parsing
|
||||
def test_lora_target_modules_single(serve_parser):
|
||||
"""Test parsing single lora-target-modules argument."""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--enable-lora", "--lora-target-modules", "o_proj"]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_multiple(serve_parser):
|
||||
"""Test parsing multiple lora-target-modules arguments."""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-lora",
|
||||
"--lora-target-modules",
|
||||
"o_proj",
|
||||
"qkv_proj",
|
||||
"down_proj",
|
||||
]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_default_none(serve_parser):
|
||||
"""Test that lora-target-modules defaults to None."""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.lora_target_modules is None
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.conftest import AudioTestAssets
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
# NOTE - the tests in this module are currently analogous to test_chat, but are
|
||||
# separated to avoid OOM killing due to module-scoped servers, since we
|
||||
# need a multimodal model for these tests.
|
||||
|
||||
# Contains a modality specific lora alongside the base model
|
||||
MULTIMODAL_MODEL_NAME = hf_api().snapshot_download(
|
||||
"microsoft/Phi-4-multimodal-instruct"
|
||||
)
|
||||
AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
|
||||
|
||||
ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def multimodal_server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"half",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"speech={AUDIO_LORA_PATH}",
|
||||
"--max-lora-rank",
|
||||
"320",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
"--trust-remote-code",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
"--default-mm-loras",
|
||||
f'{{"audio": "{AUDIO_LORA_PATH}"}}',
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MULTIMODAL_MODEL_NAME, args, max_wait_seconds=480
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def multi_modal_client(multimodal_server):
|
||||
async with multimodal_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# base model with default lora should give the same response as lora model
|
||||
"model_name",
|
||||
[MULTIMODAL_MODEL_NAME, "speech"],
|
||||
)
|
||||
async def test_default_mm_lora_chat_completions(
|
||||
model_name: str,
|
||||
multi_modal_client: openai.AsyncOpenAI,
|
||||
audio_assets: AudioTestAssets,
|
||||
):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Can you transcribe this audio?",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": audio_assets[0].url},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
chat_completion = await multi_modal_client.chat.completions.create(
|
||||
model=model_name, messages=messages, max_completion_tokens=128, temperature=0.0
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) > 0
|
||||
|
||||
message = chat_completion.choices[0].message
|
||||
assert message.content is not None and len(message.content) >= 0
|
||||
assert message.content == ACTIVE_MM_LORA_RESPONSE
|
||||
@@ -27,15 +27,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def zephyr_lora_files():
|
||||
"""Download zephyr LoRA files once per test session."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
return hf_api().snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(zephyr_lora_files):
|
||||
def server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
@@ -43,14 +35,6 @@ def server(zephyr_lora_files):
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"zephyr-lora={zephyr_lora_files}",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
@@ -67,9 +51,8 @@ async def client(server):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# first test base model, then test loras
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -91,9 +74,8 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# just test 1 lora hereafter
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -119,7 +101,7 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -145,7 +127,7 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -254,7 +236,7 @@ async def test_more_than_one_prompt_logprobs_chat(
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -297,9 +279,8 @@ async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
# just test 1 lora hereafter
|
||||
"model_name",
|
||||
[MODEL_NAME, "zephyr-lora"],
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
messages = [
|
||||
@@ -345,7 +326,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
|
||||
["HuggingFaceH4/zephyr-7b-beta"],
|
||||
)
|
||||
async def test_chat_completion_stream_options(
|
||||
client: openai.AsyncOpenAI, model_name: str
|
||||
|
||||
@@ -9,7 +9,6 @@ import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
# downloading lora to test lora requests
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
|
||||
@@ -9,12 +9,10 @@ from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
|
||||
# generation quality here
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(qwen3_lora_files):
|
||||
def server():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
@@ -22,14 +20,6 @@ def server(qwen3_lora_files):
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
f"qwen3-lora={qwen3_lora_files}",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
]
|
||||
@@ -45,12 +35,9 @@ async def client(server):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
async def test_check_models(client: openai.AsyncOpenAI):
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
served_model = models[0]
|
||||
lora_models = models[1:]
|
||||
assert served_model.id == MODEL_NAME
|
||||
assert served_model.root == MODEL_NAME
|
||||
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
|
||||
assert lora_models[0].id == "qwen3-lora"
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the token-offsets request/response protocol wiring:
|
||||
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
|
||||
serialization boundary. Plain Pydantic field storage is not retested here.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _model_config() -> Mock:
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.max_model_len = 128
|
||||
return model_config
|
||||
|
||||
|
||||
def test_completion_flag_forwarded_to_tok_params():
|
||||
"""build_tok_params must forward return_token_offsets, defaulting to
|
||||
False (zero behavioral change for existing callers) and coercing JSON
|
||||
null to False via the bool() guard."""
|
||||
cfg = _model_config()
|
||||
|
||||
default = CompletionRequest(model="m", prompt="hi")
|
||||
assert default.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
on = CompletionRequest(model="m", prompt="hi", return_token_offsets=True)
|
||||
assert on.build_tok_params(cfg).return_token_offsets is True
|
||||
|
||||
null = CompletionRequest(model="m", prompt="hi", return_token_offsets=None)
|
||||
assert null.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
|
||||
def test_chat_flag_forwarded_to_tok_params():
|
||||
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
|
||||
its return_token_offsets forwarding is verified independently."""
|
||||
cfg = _model_config()
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
default = ChatCompletionRequest(model="m", messages=messages)
|
||||
assert default.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
on = ChatCompletionRequest(model="m", messages=messages, return_token_offsets=True)
|
||||
assert on.build_tok_params(cfg).return_token_offsets is True
|
||||
|
||||
null = ChatCompletionRequest(
|
||||
model="m", messages=messages, return_token_offsets=None
|
||||
)
|
||||
assert null.build_tok_params(cfg).return_token_offsets is False
|
||||
|
||||
|
||||
def test_generate_request_token_offsets_default_none():
|
||||
"""Defaults to None so existing /v1/.../render responses are unchanged."""
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=SamplingParams())
|
||||
assert req.token_offsets is None
|
||||
|
||||
|
||||
def test_generate_request_token_offsets_survive_json_round_trip():
|
||||
"""GenerateRequest crosses the disagg serialization boundary; the
|
||||
tuple[int, int] offsets must survive model_dump and re-validate."""
|
||||
req = GenerateRequest(
|
||||
token_ids=[10, 20],
|
||||
sampling_params=SamplingParams(),
|
||||
token_offsets=[(0, 1), (1, 3)],
|
||||
)
|
||||
dumped = req.model_dump()
|
||||
assert dumped["token_offsets"] == [(0, 1), (1, 3)]
|
||||
# Re-validate from the dumped dict (sampling_params doesn't round-trip
|
||||
# cleanly via dump, so re-inject a fresh instance).
|
||||
again = GenerateRequest.model_validate(
|
||||
{**dumped, "sampling_params": SamplingParams()}
|
||||
)
|
||||
assert again.token_offsets == [(0, 1), (1, 3)]
|
||||
@@ -1,125 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
|
||||
from vllm.entrypoints.serve.middleware.authenticate import (
|
||||
GUARDED_PREFIX,
|
||||
AuthenticationMiddleware,
|
||||
)
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
ABORT_PATH = "/inference/v1/abort_requests"
|
||||
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
|
||||
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
*,
|
||||
tokens_only: bool = False,
|
||||
enable_scale_out: bool = False,
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(
|
||||
tokens_only=tokens_only,
|
||||
enable_scale_out=enable_scale_out,
|
||||
enable_fault_tolerance=False,
|
||||
)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
|
||||
@pytest.mark.parametrize("authorized", [False, True])
|
||||
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
|
||||
app = FastAPI()
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
|
||||
headers = {"Authorization": "Bearer secret"} if authorized else {}
|
||||
client = TestClient(app)
|
||||
response = client.post(path, content="not JSON", headers=headers)
|
||||
|
||||
assert response.status_code == (403 if authorized else 401)
|
||||
if authorized:
|
||||
assert "--enable-scale-out" in response.json()["error"]["message"]
|
||||
assert response.json()["error"]["type"] == "PermissionDeniedError"
|
||||
assert path not in app.openapi()["paths"]
|
||||
assert client.post("/unknown", headers=headers).status_code == 404
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert [
|
||||
record.levelno
|
||||
for record in caplog.records
|
||||
if "--enable-scale-out" in record.message
|
||||
] == [logging.INFO]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens_only", [False, True])
|
||||
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(
|
||||
tokens_only=tokens_only, enable_scale_out=not tokens_only
|
||||
)
|
||||
register_scale_out_api_routers(app, ("generate",))
|
||||
response = TestClient(app).post(
|
||||
GENERATE_PATH,
|
||||
content="not JSON",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert response.json()["detail"][0]["type"] == "json_invalid"
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
paths = registered_paths(("generate",), enable_scale_out=True)
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
|
||||
|
||||
|
||||
def test_abort_route_without_tokens_only_requires_api_key():
|
||||
"""A decode pool that loads a tokenizer serves generate without
|
||||
--tokens-only and still needs to abort, but only behind --api-key."""
|
||||
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
|
||||
|
||||
assert ABORT_PATH in paths
|
||||
assert ABORT_PATH.startswith(GUARDED_PREFIX)
|
||||
assert UNAUTHENTICATED_ABORT_PATH not in paths
|
||||
|
||||
|
||||
def test_unauthenticated_abort_route_is_not_registered_twice():
|
||||
"""A pre-existing /abort_requests route must not be shadowed by a second
|
||||
registration from the scale-out router."""
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(tokens_only=True)
|
||||
|
||||
@app.post(UNAUTHENTICATED_ABORT_PATH)
|
||||
async def existing_abort_requests():
|
||||
return None
|
||||
|
||||
attach_router(app)
|
||||
|
||||
paths = [route.path for route in app.routes]
|
||||
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
|
||||
assert ABORT_PATH in paths
|
||||
@@ -1,419 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Roundtrip tests for multimodal serde used by the
|
||||
token_in_token_out generate endpoint.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
extract_mm_features,
|
||||
merge_mm_kwargs_items,
|
||||
mm_kwargs_from_features,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
decode_mm_kwargs_item,
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.inputs import mm_input
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFieldElem,
|
||||
MultiModalFlatField,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_roundtrip():
|
||||
"""Full roundtrip test with all three field types and multiple dtypes."""
|
||||
e1 = MultiModalFieldElem(
|
||||
data=torch.zeros(1000, dtype=torch.bfloat16),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
e2 = MultiModalFieldElem(
|
||||
data=torch.ones(100, dtype=torch.int32),
|
||||
field=MultiModalSharedField(batch_size=4),
|
||||
)
|
||||
e3 = MultiModalFieldElem(
|
||||
data=torch.randn(20, dtype=torch.float32),
|
||||
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
|
||||
)
|
||||
|
||||
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
# Encoded result is a base64 string
|
||||
assert isinstance(encoded, str)
|
||||
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
|
||||
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
|
||||
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
|
||||
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
|
||||
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
|
||||
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
|
||||
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_none_data():
|
||||
"""Roundtrip with None data field."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=None,
|
||||
field=MultiModalSharedField(batch_size=2),
|
||||
)
|
||||
item = MultiModalKwargsItem({"empty": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
assert decoded["empty"].data is None
|
||||
assert isinstance(decoded["empty"].field, MultiModalSharedField)
|
||||
|
||||
|
||||
def test_mm_kwargs_item_nested_tensors():
|
||||
"""Roundtrip with nested tensor data."""
|
||||
nested = [torch.randn(3, 4), torch.randn(5, 4)]
|
||||
elem = MultiModalFieldElem(
|
||||
data=nested,
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"nested": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
decoded = decode_mm_kwargs_item(encoded)
|
||||
|
||||
decoded_data = decoded["nested"].data
|
||||
assert len(decoded_data) == 2
|
||||
assert torch.equal(nested[0], decoded_data[0])
|
||||
assert torch.equal(nested[1], decoded_data[1])
|
||||
|
||||
|
||||
def test_mm_features_with_kwargs_data():
|
||||
"""Test that MultiModalFeatures can carry serialized tensor data."""
|
||||
elem = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
item = MultiModalKwargsItem({"pixel_values": elem})
|
||||
encoded = encode_mm_kwargs_item(item)
|
||||
|
||||
features = MultiModalFeatures(
|
||||
mm_hashes={"image": ["abc123"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
|
||||
kwargs_data={"image": [encoded]},
|
||||
)
|
||||
|
||||
# JSON roundtrip
|
||||
json_str = features.model_dump_json()
|
||||
features2 = MultiModalFeatures.model_validate_json(json_str)
|
||||
|
||||
assert features2.mm_hashes == {"image": ["abc123"]}
|
||||
assert features2.kwargs_data is not None
|
||||
assert len(features2.kwargs_data["image"]) == 1
|
||||
|
||||
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
|
||||
assert torch.equal(elem.data, decoded["pixel_values"].data)
|
||||
|
||||
|
||||
def _image_engine_input(
|
||||
*,
|
||||
pixel_keep_on_cpu: bool = False,
|
||||
grid_keep_on_cpu: bool = True,
|
||||
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
|
||||
pixel_values = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
|
||||
)
|
||||
image_grid_thw = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
|
||||
)
|
||||
item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
|
||||
)
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_kwargs={"image": [item]},
|
||||
mm_hashes={"image": ["abc123"]},
|
||||
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
|
||||
)
|
||||
return engine_input, pixel_values, image_grid_thw
|
||||
|
||||
|
||||
def test_render_extracts_metadata_fields_separately():
|
||||
"""Render metadata contains CPU fields but excludes encoder inputs."""
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
|
||||
features = extract_mm_features(engine_input)
|
||||
|
||||
assert features is not None
|
||||
assert features.mm_metadata is not None
|
||||
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
|
||||
assert features.mm_metadata is not None
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert set(metadata) == {"image_grid_thw"}
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
assert features.kwargs_data is not None
|
||||
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
|
||||
assert set(full_data) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
|
||||
|
||||
|
||||
def test_extract_without_kwargs_keeps_layout():
|
||||
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
|
||||
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
|
||||
|
||||
full = extract_mm_features(engine_input)
|
||||
layout = extract_mm_features(engine_input, mm_payload="none")
|
||||
|
||||
assert full is not None and layout is not None
|
||||
assert layout.kwargs_data is None
|
||||
assert layout.mm_metadata is None
|
||||
assert layout.mm_hashes == full.mm_hashes
|
||||
assert layout.mm_placeholders == full.mm_placeholders
|
||||
|
||||
|
||||
_MESSAGES = [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_cls", "body"),
|
||||
[
|
||||
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
|
||||
(CompletionRequest, {"model": "m", "prompt": "hi"}),
|
||||
(ResponsesRequest, {"model": "m", "input": "hi"}),
|
||||
],
|
||||
)
|
||||
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
|
||||
"""Clients of the deprecated bool keep getting layout-only responses."""
|
||||
assert request_cls.model_validate(body).mm_payload == "tensors"
|
||||
|
||||
for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
|
||||
request = request_cls.model_validate(
|
||||
{**body, "return_mm_kwargs": return_mm_kwargs}
|
||||
)
|
||||
assert request.mm_payload == expected
|
||||
|
||||
explicit = request_cls.model_validate(
|
||||
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
|
||||
)
|
||||
assert explicit.mm_payload == "tensors"
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
request_cls.model_validate({**body, "mm_payload": "source"})
|
||||
|
||||
|
||||
def test_extract_includes_declared_placeholder_metadata_fields():
|
||||
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
|
||||
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
|
||||
grid_keep_on_cpu=False
|
||||
)
|
||||
|
||||
features = extract_mm_features(
|
||||
engine_input,
|
||||
metadata_fields_for=lambda modality: (
|
||||
{"image_grid_thw"} if modality == "image" else set()
|
||||
),
|
||||
)
|
||||
|
||||
assert features is not None
|
||||
assert features.mm_metadata is not None
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert set(metadata) == {"image_grid_thw"}
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_legacy_kwargs_only_generate_keeps_full_payload():
|
||||
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
assert rendered.kwargs_data is not None
|
||||
|
||||
features = MultiModalFeatures(
|
||||
mm_hashes=rendered.mm_hashes,
|
||||
mm_placeholders=rendered.mm_placeholders,
|
||||
kwargs_data=rendered.kwargs_data,
|
||||
)
|
||||
merged = mm_kwargs_from_features(features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(item["pixel_values"].data, pixel_values.data)
|
||||
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_metadata_only_generate_requires_ec_transfer_params():
|
||||
engine_input, _, _ = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": {
|
||||
"mm_hashes": rendered.mm_hashes,
|
||||
"mm_placeholders": {
|
||||
modality: [p.model_dump() for p in ranges]
|
||||
for modality, ranges in rendered.mm_placeholders.items()
|
||||
},
|
||||
"mm_metadata": rendered.mm_metadata,
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
GenerateRequest.model_validate(payload)
|
||||
raise AssertionError("expected metadata-only request without EC to fail")
|
||||
except ValidationError as exc:
|
||||
assert "ec_transfer_params" in str(exc)
|
||||
|
||||
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
|
||||
request = GenerateRequest.model_validate(payload)
|
||||
assert request.features is not None
|
||||
assert request.features.kwargs_data is None
|
||||
merged = mm_kwargs_from_features(request.features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"image_grid_thw"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modality", ["image", "video"])
|
||||
@pytest.mark.parametrize(
|
||||
("has_metadata", "has_ec"),
|
||||
[(True, False), (True, True), (False, False)],
|
||||
ids=["metadata-without-ec", "metadata-with-ec", "cache-hit-without-ec"],
|
||||
)
|
||||
def test_mixed_items_require_ec_only_for_metadata_only(
|
||||
modality: str, has_metadata: bool, has_ec: bool
|
||||
):
|
||||
"""A full item must not bypass the EC requirement for another item."""
|
||||
engine_input, _, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
features = rendered.model_dump()
|
||||
metadata = (
|
||||
encode_mm_kwargs_item(
|
||||
MultiModalKwargsItem({f"{modality}_grid_thw": image_grid_thw})
|
||||
)
|
||||
if has_metadata
|
||||
else None
|
||||
)
|
||||
second_item = {
|
||||
"mm_hashes": "second-item",
|
||||
"mm_placeholders": {"offset": 2, "length": 1},
|
||||
"kwargs_data": None,
|
||||
"mm_metadata": metadata,
|
||||
}
|
||||
for key, value in second_item.items():
|
||||
features[key].setdefault(modality, []).append(value)
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": features,
|
||||
}
|
||||
if has_ec:
|
||||
payload["ec_transfer_params"] = {"second-item": {"peer_host": "10.0.0.1"}}
|
||||
|
||||
if has_metadata and not has_ec:
|
||||
with pytest.raises(ValidationError, match="ec_transfer_params"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
else:
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_kwargs_and_metadata_generate_does_not_require_ec():
|
||||
engine_input, pixel_values, image_grid_thw = _image_engine_input()
|
||||
rendered = extract_mm_features(engine_input)
|
||||
assert rendered is not None
|
||||
request = GenerateRequest.model_validate(
|
||||
{
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 1},
|
||||
"features": rendered.model_dump(),
|
||||
}
|
||||
)
|
||||
assert request.features is not None
|
||||
merged = mm_kwargs_from_features(request.features)
|
||||
item = merged["image"][0]
|
||||
assert item is not None
|
||||
assert set(item) == {"pixel_values", "image_grid_thw"}
|
||||
assert torch.equal(item["pixel_values"].data, pixel_values.data)
|
||||
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_cache_hit_null_kwargs_keeps_metadata_slot():
|
||||
pixel_values = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3, dtype=torch.float32),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
image_grid_thw = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=True),
|
||||
)
|
||||
item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
|
||||
)
|
||||
engine_input = mm_input(
|
||||
prompt_token_ids=[1, 2, 3, 4],
|
||||
mm_kwargs={"image": [item, None]},
|
||||
mm_hashes={"image": ["hit", "cached"]},
|
||||
mm_placeholders={
|
||||
"image": [
|
||||
PlaceholderRange(offset=0, length=1),
|
||||
PlaceholderRange(offset=2, length=1),
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
features = extract_mm_features(engine_input)
|
||||
assert features is not None
|
||||
assert features.kwargs_data is not None
|
||||
assert features.mm_metadata is not None
|
||||
assert features.kwargs_data["image"][1] is None
|
||||
assert features.mm_metadata["image"][1] is None
|
||||
assert len(features.mm_metadata["image"]) == 2
|
||||
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
|
||||
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
|
||||
|
||||
|
||||
def test_features_alignment_rejects_length_mismatch():
|
||||
try:
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
)
|
||||
except ValidationError:
|
||||
return
|
||||
raise AssertionError("expected placeholder length mismatch to fail")
|
||||
|
||||
|
||||
def test_metadata_can_replace_or_extend_full_mm_data():
|
||||
"""Metadata-only requests retain the fields required by the model."""
|
||||
metadata_elem = MultiModalFieldElem(
|
||||
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
|
||||
field=MultiModalBatchedField(keep_on_cpu=True),
|
||||
)
|
||||
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
|
||||
pixel_elem = MultiModalFieldElem(
|
||||
data=torch.randn(5, 3),
|
||||
field=MultiModalBatchedField(),
|
||||
)
|
||||
full_item = MultiModalKwargsItem(
|
||||
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
|
||||
)
|
||||
|
||||
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
|
||||
merged = merge_mm_kwargs_items(full_item, metadata_item)
|
||||
assert merged is not None
|
||||
assert set(merged) == {"pixel_values", "image_grid_thw"}
|
||||
@@ -1,266 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the token_in_token_out request/response protocol.
|
||||
|
||||
These tests intentionally avoid spinning up a server — they exercise the
|
||||
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
||||
fail loudly if the validator semantics ever drift.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateStreamResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _base_payload() -> dict:
|
||||
return {"token_ids": [1, 2, 3], "sampling_params": {}}
|
||||
|
||||
|
||||
def test_omitted_max_tokens_is_not_provided():
|
||||
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
||||
server can apply its own default instead of the dataclass 16."""
|
||||
req = GenerateRequest.model_validate(_base_payload())
|
||||
# SamplingParams' dataclass default leaks through the parsed instance —
|
||||
# this is exactly the bug the server-side defaulting works around.
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_explicit_max_tokens_is_provided():
|
||||
"""Even when the client picks the same value as the dataclass default,
|
||||
it must register as explicitly set so the server won't override it."""
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"max_tokens": 16}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
payload["sampling_params"] = {"max_tokens": 256}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 256
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_other_fields_tracked_independently():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.0}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_json_roundtrip_preserves_provided_keys():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.5}
|
||||
req = GenerateRequest.model_validate_json(json.dumps(payload))
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_internal_instance_construction_treats_all_as_provided():
|
||||
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
||||
``SamplingParams`` instance, every field is considered explicitly set
|
||||
so server-side defaulting can't clobber values resolved upstream."""
|
||||
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
# And keys we never touched should also count as provided in this path.
|
||||
assert req.is_sampling_param_provided("top_p")
|
||||
|
||||
|
||||
def test_multimodal_features_reject_mismatched_parallel_fields():
|
||||
with pytest.raises(ValueError, match="same length"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
)
|
||||
|
||||
|
||||
def test_multimodal_features_reject_overlapping_placeholders():
|
||||
with pytest.raises(ValueError, match="non-overlapping"):
|
||||
MultiModalFeatures(
|
||||
mm_hashes={"image": ["a", "b"]},
|
||||
mm_placeholders={
|
||||
"image": [
|
||||
PlaceholderRangeInfo(offset=1, length=2),
|
||||
PlaceholderRangeInfo(offset=2, length=2),
|
||||
]
|
||||
},
|
||||
kwargs_data={"image": ["a", "b"]},
|
||||
)
|
||||
|
||||
|
||||
def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
with pytest.raises(ValueError, match="within the token_ids sequence"):
|
||||
GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(),
|
||||
features=MultiModalFeatures(
|
||||
mm_hashes={"image": ["a"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_parser_kwargs_rejects_unknown_keys():
|
||||
"""The engine splats these into its reasoning parser's constructor, so a
|
||||
client must not be able to pass arbitrary keyword arguments."""
|
||||
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
|
||||
with pytest.raises(ValidationError, match="tokenizer"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_rust_render_sampling_params_without_nulls_validate():
|
||||
"""Rust render omits unset sampling fields; Python generate rejects nulls."""
|
||||
sampling_params = {"max_tokens": 64, "watermarking": True}
|
||||
request = GenerateRequest.model_validate(
|
||||
{"token_ids": [1, 2, 3], "sampling_params": sampling_params}
|
||||
)
|
||||
assert request.sampling_params.max_tokens == 64
|
||||
|
||||
with pytest.raises(ValidationError, match="temperature"):
|
||||
GenerateRequest.model_validate(
|
||||
{
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {**sampling_params, "temperature": None},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_output_mode_defaults_to_tokens():
|
||||
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_output_mode_text_is_accepted():
|
||||
payload = {**_base_payload(), "output_mode": "text"}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
|
||||
def test_output_mode_rejects_unsupported_levels(output_mode):
|
||||
"""Levels the server does not implement fail instead of falling back to
|
||||
tokens which would answer 200 with the wrong response shape."""
|
||||
payload = {**_base_payload(), "output_mode": output_mode}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_text_mode_rejects_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "text",
|
||||
}
|
||||
with pytest.raises(ValidationError, match="detokenize"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_tokens_mode_allows_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "tokens",
|
||||
}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_response_without_output_mode_parses_as_tokens():
|
||||
"""Existing clients and older servers never send output_mode."""
|
||||
parsed = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensResponse)
|
||||
assert parsed.output_mode == "tokens"
|
||||
|
||||
|
||||
def test_stream_chunk_without_output_mode_parses_as_tokens():
|
||||
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_and_chunk_dispatch_on_output_mode():
|
||||
response = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert isinstance(chunk, GenerateTextStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_requires_text_on_every_choice():
|
||||
with pytest.raises(ValidationError, match="text"):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0}]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
|
||||
def test_response_rejects_unknown_output_mode(output_mode):
|
||||
with pytest.raises(ValidationError):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": output_mode, "choices": []}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_accept_responses_without_output_mode():
|
||||
DerenderChatRequest.model_validate(
|
||||
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
|
||||
)
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{
|
||||
"stream": True,
|
||||
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_reject_text_responses():
|
||||
"""Text responses are already detokenized with stop strings applied, so
|
||||
re-decoding their token_ids would silently undo the truncation."""
|
||||
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatRequest.model_validate({"generate_response": text})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
|
||||
|
||||
def test_tokens_response_echoes_output_mode_without_text():
|
||||
dumped = GenerateTokensResponse(
|
||||
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
|
||||
).model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for multimodal features through the /inference/v1/generate endpoint.
|
||||
|
||||
Mirrors test_serving_tokens.py but exercises the multimodal piping
|
||||
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
|
||||
/generate -> /detokenize path. Intentionally avoids running the HF
|
||||
processor in the pytest parent process to keep os.fork() in other
|
||||
tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
RENDER_ENDPOINT = "/v1/chat/completions/render"
|
||||
DETOKENIZE_ENDPOINT = "/detokenize"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_image():
|
||||
return Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_to_generate_roundtrip(client, test_image):
|
||||
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
|
||||
|
||||
All preprocessing and detokenization happens in the server subprocess;
|
||||
the pytest parent never imports transformers or touches torch tensors.
|
||||
"""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What color is this image? Answer in one word.",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
|
||||
render_resp.raise_for_status()
|
||||
render_data = render_resp.json()
|
||||
|
||||
# Validate render output structure: keys exist and values are non-empty
|
||||
# and well-typed.
|
||||
assert "token_ids" in render_data
|
||||
assert isinstance(render_data["token_ids"], list)
|
||||
assert len(render_data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in render_data["token_ids"])
|
||||
|
||||
assert "features" in render_data
|
||||
features = render_data["features"]
|
||||
assert features is not None
|
||||
assert isinstance(features, dict)
|
||||
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) and h for h in image_hashes)
|
||||
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert isinstance(p.get("offset"), int)
|
||||
assert isinstance(p.get("length"), int)
|
||||
assert p["length"] > 0
|
||||
|
||||
assert "kwargs_data" in features
|
||||
assert "image" in features["kwargs_data"]
|
||||
assert len(features["kwargs_data"]["image"]) > 0
|
||||
|
||||
# Build generate request from render output
|
||||
generate_payload = render_data
|
||||
generate_payload["sampling_params"] = {
|
||||
"max_tokens": 10,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
|
||||
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
assert isinstance(gen_data["choices"], list)
|
||||
assert len(gen_data["choices"]) >= 1
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert isinstance(choice["token_ids"], list)
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in choice["token_ids"])
|
||||
|
||||
detok_resp = await client.post(
|
||||
DETOKENIZE_ENDPOINT,
|
||||
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
|
||||
)
|
||||
detok_resp.raise_for_status()
|
||||
detok_data = detok_resp.json()
|
||||
assert "prompt" in detok_data
|
||||
text = detok_data["prompt"]
|
||||
assert isinstance(text, str)
|
||||
assert len(text) > 0
|
||||
assert "red" in text.lower(), (
|
||||
f"Expected model to identify the red image, got: {text!r}"
|
||||
)
|
||||
|
||||
|
||||
def _collapse_placeholders(
|
||||
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
|
||||
) -> list[int]:
|
||||
"""Undo placeholder expansion: each expanded range becomes one token.
|
||||
|
||||
``/generate`` with ``content_parts`` expects unexpanded placeholders and
|
||||
expands them itself, so the roundtrip must start from a collapsed prompt.
|
||||
"""
|
||||
ranges = sorted(
|
||||
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
|
||||
)
|
||||
collapsed: list[int] = []
|
||||
pos = 0
|
||||
for offset, length in ranges:
|
||||
collapsed.extend(token_ids[pos:offset])
|
||||
collapsed.append(token_ids[offset])
|
||||
pos = offset + length
|
||||
collapsed.extend(token_ids[pos:])
|
||||
return collapsed
|
||||
|
||||
|
||||
async def _render_image_prompt(client, data_url: str) -> dict:
|
||||
render_resp = await client.post(
|
||||
RENDER_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What color is this?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
render_resp.raise_for_status()
|
||||
return render_resp.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_generates_tokens(client, test_image):
|
||||
"""Unexpanded prompt + content_parts must expand exactly like /render."""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
render_data = await _render_image_prompt(client, data_url)
|
||||
expanded = render_data["token_ids"]
|
||||
mm_placeholders = render_data["features"]["mm_placeholders"]
|
||||
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
|
||||
assert len(unexpanded) < len(expanded)
|
||||
|
||||
gen_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": unexpanded,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert gen_data["prompt_token_ids"] == expanded
|
||||
assert gen_data["mm_placeholders"] == mm_placeholders
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_streaming(client, test_image):
|
||||
"""content_parts should work with streaming."""
|
||||
import json as json_mod
|
||||
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
render_data = await _render_image_prompt(client, data_url)
|
||||
expanded = render_data["token_ids"]
|
||||
mm_placeholders = render_data["features"]["mm_placeholders"]
|
||||
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
|
||||
|
||||
async with client.stream(
|
||||
"POST",
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": unexpanded,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
|
||||
"stream": True,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json_mod.loads(line[6:]))
|
||||
|
||||
assert len(chunks) > 0
|
||||
prompt_metadata_chunks = [
|
||||
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
|
||||
]
|
||||
assert len(prompt_metadata_chunks) == 1
|
||||
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
|
||||
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders
|
||||
@@ -1,843 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
ABORT_ENDPOINT = "/inference/v1/abort_requests"
|
||||
|
||||
|
||||
def get_vocab_size(model_name):
|
||||
config = ModelConfig(
|
||||
model=model_name,
|
||||
seed=0,
|
||||
dtype="bfloat16",
|
||||
)
|
||||
return config.get_vocab_size()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def messages():
|
||||
return [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "How many countries are in the EU?"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(request):
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
|
||||
# 1 ULP when the batch dimension (M) changes, because different M
|
||||
# values cause the Tensile backend to select different tile
|
||||
# configurations with different fp32 accumulation orders. With
|
||||
# prefix caching, cache-miss prefills compute all tokens in one
|
||||
# pass (large M) while cache-hit requests compute only the
|
||||
# uncached suffix (small M), seeding a divergence that amplifies
|
||||
# through the residual stream and flips argmax tokens.
|
||||
# See: https://github.com/vllm-project/vllm/issues/33123
|
||||
#
|
||||
# Either disable prefix caching entirely, or enable it with
|
||||
# --deterministic-prefix-caching which forces cache-miss prefills
|
||||
# to split at block boundaries so the suffix GEMM shape is always
|
||||
# identical regardless of cache state.
|
||||
#
|
||||
# Option A: disable prefix caching
|
||||
"--no-enable-prefix-caching",
|
||||
#
|
||||
# Option B: deterministic prefix caching
|
||||
# "--enable-prefix-caching",
|
||||
# "--deterministic-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
extra_args = getattr(request, "param", None)
|
||||
if extra_args is not None:
|
||||
args = args + (
|
||||
list(extra_args)
|
||||
if isinstance(extra_args, (list, tuple))
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_endpoint(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
assert data["choices"][0].get("sampling_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
|
||||
"""Explicit null max_tokens must not skip the min_tokens bound."""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": None, "min_tokens": 2147483648},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "min_tokens" in resp.json()["error"]["message"]
|
||||
|
||||
followup = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 4},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=followup)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="sampling mask output is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server",
|
||||
[["--return-sampling-mask", "--logprobs-mode", "processed_logprobs"]],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_generate_sampling_mask(client):
|
||||
top_k = 5
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.8,
|
||||
"top_k": top_k,
|
||||
"top_p": 0.9,
|
||||
"ignore_eos": True,
|
||||
"seed": 0,
|
||||
"logprobs": top_k,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
choice = resp.json()["choices"][0]
|
||||
|
||||
token_ids = choice["token_ids"]
|
||||
sampling_mask = choice["sampling_mask"]
|
||||
assert sampling_mask is not None
|
||||
assert len(token_ids) == len(sampling_mask)
|
||||
|
||||
vocab_size = get_vocab_size(MODEL_NAME)
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
|
||||
assert support
|
||||
assert len(support) == len(set(support))
|
||||
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
|
||||
assert token_id in support
|
||||
# processed_logprobs: exactly the support carries finite probability.
|
||||
for top in entry["top_logprobs"]:
|
||||
in_support = top["token_id"] in support
|
||||
assert in_support == (top["logprob"] > -9999.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_defaults_max_tokens_when_omitted(client):
|
||||
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
|
||||
|
||||
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
|
||||
defaulting was wired up, every request that didn't set ``max_tokens``
|
||||
truncated mid-generation. The server should now fill it in from
|
||||
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
completion_tokens = len(data["choices"][0]["token_ids"])
|
||||
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
|
||||
# should get ~1021 tokens of output (capped at max_model_len boundary).
|
||||
assert completion_tokens > 16, (
|
||||
f"expected server-side default to exceed the legacy 16-token cap, "
|
||||
f"got {completion_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
|
||||
"""Regression: ``n > 1`` must return ``n`` choices.
|
||||
|
||||
Non-streaming requests kept ``SamplingParams``' default output kind,
|
||||
``CUMULATIVE``, so only the sequences updated during the last engine step
|
||||
reached the response and the others were silently dropped.
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_stream(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": True,
|
||||
}
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload_str = line[len("data: ") :]
|
||||
if payload_str == "[DONE]":
|
||||
break
|
||||
chunks.append(json.loads(payload_str))
|
||||
|
||||
assert len(chunks) > 0
|
||||
# Every chunk has choices with token_ids
|
||||
all_token_ids = []
|
||||
for chunk in chunks:
|
||||
assert "choices" in chunk
|
||||
assert len(chunk["choices"]) == 1
|
||||
choice = chunk["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
all_token_ids.extend(choice["token_ids"])
|
||||
|
||||
# Last chunk should have a finish_reason
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
|
||||
# Streaming should produce the same tokens as non-streaming
|
||||
non_stream_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
non_stream_data = non_stream_resp.json()
|
||||
# Just verify we got the right number of tokens
|
||||
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
|
||||
async def test_generate_logprobs(client, logprobs_value):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
"logprobs": logprobs_value,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
choice = data["choices"][0]
|
||||
assert choice["logprobs"] is not None
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
assert len(logprobs_content) == len(choice["token_ids"])
|
||||
for entry in logprobs_content:
|
||||
assert "logprob" in entry
|
||||
assert len(entry["top_logprobs"]) >= 1
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
rows = [[33], [44, 100], [55]]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": rows,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": sampling_params,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (3, 2)
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
|
||||
assert np.isneginf(scores[[0, 2], 1]).all()
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": rows}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "scored rows" in resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
for ignore_eos in [True, False]:
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
# NOTE coordinator will set this to skip detokenization
|
||||
"detokenize": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
gen_token_ids = generate_data["choices"][0]["token_ids"]
|
||||
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
if ignore_eos:
|
||||
# When ignoring EOS, only compare up to the first EOS token
|
||||
# Post-EOS generation is undefined and may differ
|
||||
eos_tokens = {
|
||||
tokenizer.eos_token_id,
|
||||
*tokenizer.extra_special_tokens_ids,
|
||||
}
|
||||
# Find first EOS in generated tokens
|
||||
eos_pos = None
|
||||
for i, tid in enumerate(gen_token_ids):
|
||||
if tid in eos_tokens:
|
||||
eos_pos = i
|
||||
break
|
||||
if eos_pos is not None:
|
||||
gen_token_ids_truncated = gen_token_ids[:eos_pos]
|
||||
generate_res = tokenizer.decode(
|
||||
gen_token_ids_truncated, skip_special_tokens=True
|
||||
)
|
||||
# Truncate completions_res to same length for comparison
|
||||
completions_res = completions_res[: len(generate_res)]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_string_workflow(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
# stop strings are only supported when detokenize is True.
|
||||
"stop": ["27 member"],
|
||||
},
|
||||
# TODO stream test is much more interesting
|
||||
"stream": False,
|
||||
}
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_resp.raise_for_status()
|
||||
|
||||
payload["sampling_params"]["stop"] = None
|
||||
generate_resp = await client.post(
|
||||
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
|
||||
)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
# NOTE This is under the responsibility of the coordinator
|
||||
# stop_checker = StopChecker(
|
||||
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
|
||||
# )
|
||||
stop_str, truncate_to = check_stop_strings(
|
||||
generate_res, len(generate_res), ["27 member"], False
|
||||
)
|
||||
assert stop_str == "27 member"
|
||||
# abort request that hit stop string (requires tokens-only mode)
|
||||
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
|
||||
# res.raise_for_status()
|
||||
generate_res = generate_res[:truncate_to]
|
||||
|
||||
# Get stop_str response from chat completions
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"stop": ["27 member"],
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_stop_strings_rejected_when_tokens_only(client):
|
||||
"""--tokens-only forces detokenize=False after request validation, so
|
||||
stop strings must be rejected explicitly rather than silently ignored."""
|
||||
sampling_params = {"max_tokens": 5, "stop": ["never"]}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": sampling_params,
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "stop strings" in resp.json()["error"]["message"]
|
||||
|
||||
sampling_params["stop"] = None
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server",
|
||||
[
|
||||
[
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
"Alice=charent/self_cognition_Alice",
|
||||
"Bob=charent/self_cognition_Bob",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
# Verify adapters are listed
|
||||
models_resp = await client.get("/v1/models")
|
||||
models_resp.raise_for_status()
|
||||
models = {m["id"] for m in models_resp.json().get("data", [])}
|
||||
assert {"Alice", "Bob"}.issubset(models)
|
||||
|
||||
# Generate using a LoRA adapter by specifying its name as the model
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
def _structured_chat_body(messages, **overrides) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 64,
|
||||
"temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"count": {"type": "integer"}},
|
||||
"required": ["count"],
|
||||
},
|
||||
},
|
||||
},
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
|
||||
render_resp = await client.post("/v1/chat/completions/render", json=body)
|
||||
render_resp.raise_for_status()
|
||||
generate_request = render_resp.json()
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
|
||||
generate_resp.raise_for_status()
|
||||
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
|
||||
# <think> tokens, so the engine gates structured outputs differently when
|
||||
# either reasoning field is dropped between render and generate.
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
|
||||
"""With thinking on, structured outputs must wait for the reasoning to
|
||||
end on generate, the same as on chat."""
|
||||
body = _structured_chat_body(messages, return_token_ids=True)
|
||||
chat_resp = await client.post("/v1/chat/completions", json=body)
|
||||
chat_resp.raise_for_status()
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is False
|
||||
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
|
||||
)
|
||||
async def test_render_then_generate_forwards_reasoning_ended(
|
||||
client, tokenizer, messages
|
||||
):
|
||||
"""`include_reasoning=false` constrains from the first token on chat, so
|
||||
generate must too, even though the engine's parser expects thinking."""
|
||||
body = _structured_chat_body(messages, include_reasoning=False)
|
||||
|
||||
generate_request, token_ids = await _render_then_generate(client, body)
|
||||
|
||||
assert generate_request["reasoning_ended"] is True
|
||||
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
|
||||
assert isinstance(output["count"], int)
|
||||
|
||||
|
||||
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
|
||||
return tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
|
||||
def _text_mode_payload(token_ids, **sampling_params) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
|
||||
"output_mode": "text",
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == "text"
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
chat_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
assert data["choices"][0]["text"] == chat_content
|
||||
assert data["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
payload = _text_mode_payload(token_ids)
|
||||
|
||||
non_stream = await client.post(GEN_ENDPOINT, json=payload)
|
||||
non_stream.raise_for_status()
|
||||
|
||||
chunks = []
|
||||
async with client.stream(
|
||||
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json.loads(line[len("data: ") :]))
|
||||
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
|
||||
assert streamed_text == non_stream.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
pytest.param(
|
||||
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
|
||||
),
|
||||
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
|
||||
],
|
||||
)
|
||||
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
|
||||
payload = {
|
||||
**_text_mode_payload([1, 2, 3], max_tokens=5),
|
||||
**overrides,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
|
||||
assert resp.status_code == 400
|
||||
assert expected in resp.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_text_mode_rejected_when_tokens_only(client):
|
||||
"""Without a tokenizer the detokenizer returns "", so the request must
|
||||
fail instead of answering 200 with empty text."""
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
|
||||
assert resp.status_code == 400
|
||||
assert "requires a tokenizer" in resp.json()["error"]["message"]
|
||||
|
||||
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
|
||||
payload["sampling_params"]["detokenize"] = False
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
|
||||
async def test_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages, output_mode
|
||||
):
|
||||
"""The final output after an abort has no new token IDs, but the
|
||||
stream must still deliver its finish reason."""
|
||||
payload = {
|
||||
**_text_mode_payload(
|
||||
_chat_prompt_token_ids(tokenizer, messages),
|
||||
max_tokens=900,
|
||||
ignore_eos=True,
|
||||
),
|
||||
"output_mode": output_mode,
|
||||
"stream": True,
|
||||
"request_id": f"{output_mode}-abort-e2e",
|
||||
}
|
||||
|
||||
chunks = []
|
||||
aborted = False
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
chunks.append(chunk)
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
|
||||
abort = await client.post(
|
||||
ABORT_ENDPOINT,
|
||||
json={"request_ids": [chunk["request_id"]]},
|
||||
)
|
||||
assert abort.status_code == 200
|
||||
aborted = True
|
||||
|
||||
assert aborted
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
|
||||
final_choice = chunks[-1]["choices"][0]
|
||||
assert final_choice["finish_reason"] == "abort"
|
||||
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
|
||||
assert generated < 900
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_abort_requests_is_served_without_tokens_only(client):
|
||||
resp = await client.post(
|
||||
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = await client.post(ABORT_ENDPOINT, json={})
|
||||
assert resp.status_code == 400
|
||||
|
||||
# The unauthenticated path is only served with --tokens-only.
|
||||
resp = await client.post(
|
||||
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own integer token id."""
|
||||
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
|
||||
)
|
||||
token_ids = {e.token_id for e in result.content[0].top_logprobs}
|
||||
assert token_ids == {262, 257}, f"got {token_ids}"
|
||||
|
||||
|
||||
def test_sampled_entry_carries_token_id_and_rank():
|
||||
"""The sampled token is identified by id, and the engine's rank survives."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[
|
||||
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
|
||||
],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 262
|
||||
assert entry.logprob == -0.1
|
||||
assert entry.rank == 1
|
||||
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
|
||||
# No tokenizer on the generate server: no string token, no bytes.
|
||||
assert not hasattr(entry, "token")
|
||||
assert not hasattr(entry, "bytes")
|
||||
|
||||
|
||||
def test_sampled_token_absent_from_topk_uses_sentinel():
|
||||
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[5],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token_id == 5
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs == []
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9)}],
|
||||
)
|
||||
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
|
||||
|
||||
|
||||
def test_logprobs_minus_one_emits_all_tokens():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
|
||||
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
|
||||
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.token == "token_id:7"
|
||||
assert entry.bytes is None
|
||||
assert all(t.bytes is None for t in entry.top_logprobs)
|
||||
|
||||
|
||||
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
tokenizer = MagicMock()
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
tokenizer.decode.assert_not_called()
|
||||
|
||||
|
||||
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
assert result.content[0].top_logprobs[0].token == "<7>"
|
||||
assert result.content[1].bytes == list(b"<9>")
|
||||
|
||||
|
||||
def test_sampled_token_outside_topk_comes_first():
|
||||
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
|
||||
all k + 1 candidates when the sampled token is outside the top k; derender
|
||||
applies each endpoint's cut (#59513)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[50],
|
||||
top_logprobs=[
|
||||
{
|
||||
50: Logprob(-3.0, rank=5),
|
||||
10: Logprob(-0.2, rank=1),
|
||||
20: Logprob(-1.0, rank=2),
|
||||
}
|
||||
],
|
||||
)
|
||||
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
|
||||
|
||||
|
||||
def test_logprob_is_required_on_the_wire():
|
||||
"""A payload missing logprob is rejected rather than read as -9999."""
|
||||
with pytest.raises(ValidationError):
|
||||
GenerateLogProb.model_validate({"token_id": 1})
|
||||
|
||||
|
||||
def test_rank_zero_from_a_nan_logprob_is_none():
|
||||
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.rank is None
|
||||
assert entry.top_logprobs[0].rank is None
|
||||
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
|
||||
|
||||
def test_text_logprobs_clamp_a_nan_logprob():
|
||||
result = ServingTokens._create_text_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
|
||||
tokenizer=MagicMock(),
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.logprob == -9999.0
|
||||
assert entry.top_logprobs[0].logprob == -9999.0
|
||||
json.dumps(result.model_dump(), allow_nan=False)
|
||||
@@ -120,7 +120,7 @@ class TestAffectedModulesUseSanitize:
|
||||
@pytest.mark.parametrize(
|
||||
"module",
|
||||
[
|
||||
"vllm.entrypoints.speech_to_text.realtime.connection",
|
||||
"vllm.entrypoints.serve.exception_handling.handlers.http",
|
||||
],
|
||||
)
|
||||
def test_module_calls_sanitize_message(self, module: str):
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import shutil
|
||||
from contextlib import suppress
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# any model with a chat template should work here
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
BADREQUEST_CASES = [
|
||||
(
|
||||
"test_rank",
|
||||
{"r": 1024},
|
||||
"is greater than max_lora_rank",
|
||||
),
|
||||
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
|
||||
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
|
||||
(
|
||||
"test_modules_to_save",
|
||||
{"modules_to_save": ["lm_head"]},
|
||||
"Unsupported modules_to_save",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[True])
|
||||
def server_with_lora_modules_json(request, qwen3_lora_files):
|
||||
# Define the json format LoRA module configurations
|
||||
lora_module_1 = {
|
||||
"name": "qwen3-lora",
|
||||
"path": qwen3_lora_files,
|
||||
"base_model_name": MODEL_NAME,
|
||||
}
|
||||
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--enforce-eager",
|
||||
# lora config below
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
json.dumps(lora_module_1),
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
"--max-num-seqs",
|
||||
"64",
|
||||
]
|
||||
|
||||
# Enable the /v1/load_lora_adapter endpoint
|
||||
envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server_with_lora_modules_json):
|
||||
async with server_with_lora_modules_json.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
served_model = models[0]
|
||||
lora_models = models[1:]
|
||||
assert served_model.id == MODEL_NAME
|
||||
assert served_model.root == MODEL_NAME
|
||||
assert served_model.parent is None
|
||||
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
|
||||
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
|
||||
assert lora_models[0].id == "qwen3-lora"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
|
||||
)
|
||||
# Ensure adapter loads before querying /models
|
||||
assert "success" in response
|
||||
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
dynamic_lora_model = models[-1]
|
||||
assert dynamic_lora_model.root == qwen3_lora_files
|
||||
assert dynamic_lora_model.parent == MODEL_NAME
|
||||
assert dynamic_lora_model.id == "qwen3-lora-3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_with_same_name_replaces_inplace(
|
||||
client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
|
||||
):
|
||||
"""Test that loading a LoRA adapter with the same name replaces it inplace."""
|
||||
adapter_name = "replaceable-adapter"
|
||||
messages = [
|
||||
{"content": "Follow the instructions to make animal noises", "role": "system"},
|
||||
{"content": "Make your favorite animal noise.", "role": "user"},
|
||||
]
|
||||
|
||||
# Load LoRA that makes model meow
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
completion = await client.chat.completions.create(
|
||||
model=adapter_name,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
assert "Meow Meow Meow" in completion.choices[0].message.content
|
||||
|
||||
# Load LoRA that makes model woof
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={
|
||||
"lora_name": adapter_name,
|
||||
"lora_path": qwen3_woofing_lora_files,
|
||||
"load_inplace": True,
|
||||
},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
completion = await client.chat.completions.create(
|
||||
model=adapter_name,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
)
|
||||
assert "Woof Woof Woof" in completion.choices[0].message.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_with_load_inplace_false_errors(
|
||||
client: openai.AsyncOpenAI, qwen3_meowing_lora_files
|
||||
):
|
||||
"""Test that load_inplace=False returns an error when adapter already exists."""
|
||||
adapter_name = "test-load-inplace-false"
|
||||
|
||||
# Load LoRA adapter first time (should succeed)
|
||||
response = await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
|
||||
)
|
||||
assert "success" in response.lower()
|
||||
|
||||
# Try to load the same adapter again with load_inplace=False (should fail)
|
||||
with pytest.raises(openai.BadRequestError) as exc_info:
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={
|
||||
"lora_name": adapter_name,
|
||||
"lora_path": qwen3_meowing_lora_files,
|
||||
},
|
||||
)
|
||||
|
||||
# Verify the error message
|
||||
assert "already been loaded" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
|
||||
with pytest.raises(openai.NotFoundError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
|
||||
invalid_files = tmp_path / "invalid_files"
|
||||
invalid_files.mkdir()
|
||||
(invalid_files / "adapter_config.json").write_text("this is not json")
|
||||
|
||||
with pytest.raises(openai.InternalServerError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "invalid-json", "lora_path": str(invalid_files)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES)
|
||||
async def test_dynamic_lora_badrequests(
|
||||
client: openai.AsyncOpenAI,
|
||||
tmp_path,
|
||||
qwen3_lora_files,
|
||||
test_name: str,
|
||||
config_change: dict,
|
||||
expected_error: str,
|
||||
):
|
||||
# Create test directory
|
||||
test_dir = tmp_path / test_name
|
||||
|
||||
# Copy adapter files
|
||||
shutil.copytree(qwen3_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
with open(config_path) as f:
|
||||
adapter_config = json.load(f)
|
||||
# Apply configuration changes
|
||||
adapter_config.update(config_change)
|
||||
|
||||
# Save modified configuration
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(adapter_config, f)
|
||||
|
||||
# Test loading the adapter
|
||||
with pytest.raises(openai.InternalServerError, match=expected_error):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": test_name, "lora_path": str(test_dir)},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_lora_adapters(
|
||||
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
||||
):
|
||||
"""Validate that many loras can be dynamically registered and inferenced
|
||||
with concurrently"""
|
||||
|
||||
# This test file configures the server with --max-cpu-loras=2 and this test
|
||||
# will concurrently load 10 adapters, so it should flex the LRU cache
|
||||
async def load_and_run_adapter(adapter_name: str):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
|
||||
)
|
||||
for _ in range(3):
|
||||
await client.completions.create(
|
||||
model=adapter_name,
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
|
||||
lora_tasks = []
|
||||
for i in range(10):
|
||||
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
|
||||
|
||||
results, _ = await asyncio.wait(lora_tasks)
|
||||
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loading_invalid_adapters_does_not_break_others(
|
||||
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
|
||||
):
|
||||
invalid_files = tmp_path / "invalid_files"
|
||||
invalid_files.mkdir()
|
||||
(invalid_files / "adapter_config.json").write_text("this is not json")
|
||||
|
||||
stop_good_requests_event = asyncio.Event()
|
||||
|
||||
async def run_good_requests(client):
|
||||
# Run chat completions requests until event set
|
||||
|
||||
results = []
|
||||
|
||||
while not stop_good_requests_event.is_set():
|
||||
try:
|
||||
batch = await client.completions.create(
|
||||
model="qwen3-lora",
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
results.append(batch)
|
||||
except Exception as e:
|
||||
results.append(e)
|
||||
|
||||
return results
|
||||
|
||||
# Create task to run good requests
|
||||
good_task = asyncio.create_task(run_good_requests(client))
|
||||
|
||||
# Run a bunch of bad adapter loads
|
||||
for _ in range(25):
|
||||
with suppress(openai.NotFoundError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
|
||||
)
|
||||
for _ in range(25):
|
||||
with suppress(openai.InternalServerError):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "invalid", "lora_path": str(invalid_files)},
|
||||
)
|
||||
|
||||
# Ensure all the running requests with lora adapters succeeded
|
||||
stop_good_requests_event.set()
|
||||
results = await good_task
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
|
||||
# Ensure we can load another adapter and run it
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": "valid", "lora_path": qwen3_lora_files},
|
||||
)
|
||||
await client.completions.create(
|
||||
model="valid",
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_beam_search_with_lora_adapters(
|
||||
client: openai.AsyncOpenAI,
|
||||
tmp_path,
|
||||
qwen3_lora_files,
|
||||
):
|
||||
"""Validate that async beam search can be used with lora."""
|
||||
|
||||
async def load_and_run_adapter(adapter_name: str):
|
||||
await client.post(
|
||||
"load_lora_adapter",
|
||||
cast_to=str,
|
||||
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
|
||||
)
|
||||
for _ in range(3):
|
||||
await client.completions.create(
|
||||
model=adapter_name,
|
||||
prompt=["Hello there", "Foo bar bazz buzz"],
|
||||
max_tokens=5,
|
||||
extra_body=dict(use_beam_search=True),
|
||||
)
|
||||
|
||||
lora_tasks = []
|
||||
for i in range(3):
|
||||
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
|
||||
|
||||
results, _ = await asyncio.wait(lora_tasks)
|
||||
|
||||
for r in results:
|
||||
assert not isinstance(r, Exception), f"Got exception {r}"
|
||||
@@ -214,7 +214,6 @@ def test_request_logger_log_outputs_integration():
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_embeds=None,
|
||||
params=None,
|
||||
lora_request=None,
|
||||
)
|
||||
|
||||
request_logger.log_outputs(
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
|
||||
def add_attention_backend(server_args, attention_config):
|
||||
"""Append attention backend CLI arg if specified.
|
||||
|
||||
Args:
|
||||
server_args: List of server arguments to extend in-place.
|
||||
attention_config: Dict with 'backend' key, or None.
|
||||
|
||||
"""
|
||||
if attention_config and "backend" in attention_config:
|
||||
server_args.extend(["--attention-backend", attention_config["backend"]])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mary_had_lamb():
|
||||
path = AudioAsset("mary_had_lamb").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def winning_call():
|
||||
path = AudioAsset("winning_call").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def foscolo():
|
||||
# Test translation it->en
|
||||
path = AudioAsset("azacinto_foscolo").get_local_path()
|
||||
with open(str(path), "rb") as f:
|
||||
yield f
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def transcription_server_with_force_include_usage():
|
||||
args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--enforce-eager",
|
||||
"--enable-force-include-usage",
|
||||
"--gpu-memory-utilization",
|
||||
"0.2",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transcription_client_with_force_include_usage(
|
||||
transcription_server_with_force_include_usage,
|
||||
):
|
||||
async with (
|
||||
transcription_server_with_force_include_usage.get_async_client() as async_client
|
||||
):
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_with_enable_force_include_usage(
|
||||
transcription_client_with_force_include_usage, winning_call
|
||||
):
|
||||
res = (
|
||||
await transcription_client_with_force_include_usage.audio.transcriptions.create(
|
||||
model="openai/whisper-large-v3-turbo",
|
||||
file=winning_call,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
stream=True,
|
||||
timeout=30,
|
||||
)
|
||||
)
|
||||
|
||||
async for chunk in res:
|
||||
if not len(chunk.choices):
|
||||
# final usage sent
|
||||
usage = chunk.usage
|
||||
assert isinstance(usage, dict)
|
||||
assert usage["prompt_tokens"] > 0
|
||||
assert usage["completion_tokens"] > 0
|
||||
assert usage["total_tokens"] > 0
|
||||
else:
|
||||
assert not hasattr(chunk, "usage")
|
||||
@@ -592,13 +592,12 @@ class TestLoadWeightsExpertMapping:
|
||||
return calls
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"projs,lora_prefix",
|
||||
"projs",
|
||||
[
|
||||
(("gate_proj", "down_proj", "up_proj"), ""),
|
||||
(("w1", "w2", "w3"), ""),
|
||||
(("up_proj", "down_proj", "up_proj"), ""),
|
||||
(("up_proj", "down_proj", None), ""),
|
||||
(("gate_proj", "down_proj", "up_proj"), "base_layer."),
|
||||
("gate_proj", "down_proj", "up_proj"),
|
||||
("w1", "w2", "w3"),
|
||||
("up_proj", "down_proj", "up_proj"),
|
||||
("up_proj", "down_proj", None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -610,12 +609,11 @@ class TestLoadWeightsExpertMapping:
|
||||
],
|
||||
)
|
||||
def test_per_expert_weights_keep_all_physical_destinations(
|
||||
self, projs, lora_prefix, suffix, shape, dtype
|
||||
self, projs, suffix, shape, dtype
|
||||
):
|
||||
mapping = self._mapping(
|
||||
projs,
|
||||
num_redundant_experts=4,
|
||||
lora_base_layer_prefix=lora_prefix,
|
||||
is_gated=projs[2] is not None,
|
||||
)
|
||||
for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]:
|
||||
@@ -623,7 +621,7 @@ class TestLoadWeightsExpertMapping:
|
||||
if not proj:
|
||||
continue
|
||||
weight = torch.full(shape, logical_id + 1, dtype=dtype)
|
||||
name = f"{logical_id}.{proj}.{lora_prefix}{suffix}"
|
||||
name = f"{logical_id}.{proj}.{suffix}"
|
||||
calls = self._load(mapping, [(name, weight)], suffix)
|
||||
shards = [s for p, s in zip(projs, ("w1", "w2", "w3")) if p == proj]
|
||||
expected = [
|
||||
@@ -636,20 +634,18 @@ class TestLoadWeightsExpertMapping:
|
||||
torch.testing.assert_close(call[3], weight, rtol=0, atol=0)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"projs,fused_name,lora_prefix",
|
||||
"projs,fused_name",
|
||||
[
|
||||
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj", ""),
|
||||
(("w1", "w2", "w3"), "w13", "base_layer."),
|
||||
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj"),
|
||||
(("w1", "w2", "w3"), "w13"),
|
||||
],
|
||||
)
|
||||
def test_per_expert_fused_gate_up_keeps_both_halves(
|
||||
self, projs, fused_name, lora_prefix
|
||||
self, projs, fused_name
|
||||
):
|
||||
mapping = self._mapping(
|
||||
projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix
|
||||
)
|
||||
mapping = self._mapping(projs, num_redundant_experts=4)
|
||||
weight = torch.arange(12).reshape(4, 3)
|
||||
calls = self._load(mapping, [(f"1.{fused_name}.{lora_prefix}weight", weight)])
|
||||
calls = self._load(mapping, [(f"1.{fused_name}.weight", weight)])
|
||||
assert [call[:3] for call in calls] == [
|
||||
("w13_weight", shard, expert)
|
||||
for expert in (1, 17)
|
||||
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from collections import OrderedDict
|
||||
from importlib import reload
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def should_do_global_cleanup_after_test(request) -> bool:
|
||||
"""Allow subdirectories to skip global cleanup by overriding this fixture.
|
||||
This can provide a ~10x speedup for non-GPU unit tests since they don't need
|
||||
to initialize torch.
|
||||
"""
|
||||
return not request.node.get_closest_marker("skip_global_cleanup")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
|
||||
yield
|
||||
if should_do_global_cleanup_after_test:
|
||||
cleanup_dist_env_and_memory(shutdown_ray=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def maybe_enable_lora_dual_stream(monkeypatch: pytest.MonkeyPatch):
|
||||
if current_platform.is_cuda():
|
||||
monkeypatch.setenv("VLLM_LORA_ENABLE_DUAL_STREAM", "1")
|
||||
import vllm.lora.layers.base_linear
|
||||
|
||||
if not hasattr(vllm.lora.layers.base_linear, "lora_linear_async"):
|
||||
# Reload the module to ensure the environment variable takes effect.
|
||||
reload(vllm.lora.layers.base_linear)
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dist_init():
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
|
||||
backend = "gloo" if current_platform.is_tpu() else current_platform.dist_backend
|
||||
|
||||
with ensure_current_vllm_config():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend=backend,
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory(shutdown_ray=True)
|
||||
|
||||
|
||||
class DummyLoRAModel(nn.Sequential, SupportsLoRA):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_model(default_vllm_config) -> nn.Module:
|
||||
model = DummyLoRAModel(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", ColumnParallelLinear(764, 100)),
|
||||
("dense2", RowParallelLinear(100, 50)),
|
||||
(
|
||||
"layer1",
|
||||
nn.Sequential(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", ColumnParallelLinear(100, 10)),
|
||||
("dense2", RowParallelLinear(10, 50)),
|
||||
]
|
||||
)
|
||||
),
|
||||
),
|
||||
("act2", nn.ReLU()),
|
||||
("output", ColumnParallelLinear(50, 10)),
|
||||
("outact", nn.Sigmoid()),
|
||||
# Special handling for lm_head & sampler
|
||||
("lm_head", ParallelLMHead(32064, 10)),
|
||||
("logits_processor", LogitsProcessor(32064)),
|
||||
]
|
||||
)
|
||||
)
|
||||
model.config = MagicMock()
|
||||
model.embedding_modules = {"lm_head": "lm_head"}
|
||||
model.unpadded_vocab_size = 32064
|
||||
return model
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_model_gate_up(default_vllm_config) -> nn.Module:
|
||||
model = DummyLoRAModel(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", ColumnParallelLinear(764, 100)),
|
||||
("dense2", RowParallelLinear(100, 50)),
|
||||
(
|
||||
"layer1",
|
||||
nn.Sequential(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", ColumnParallelLinear(100, 10)),
|
||||
("dense2", RowParallelLinear(10, 50)),
|
||||
]
|
||||
)
|
||||
),
|
||||
),
|
||||
("act2", nn.ReLU()),
|
||||
("gate_up_proj", MergedColumnParallelLinear(50, [5, 5])),
|
||||
("outact", nn.Sigmoid()),
|
||||
# Special handling for lm_head & sampler
|
||||
("lm_head", ParallelLMHead(32064, 10)),
|
||||
("logits_processor", LogitsProcessor(32064)),
|
||||
]
|
||||
)
|
||||
)
|
||||
model.config = MagicMock()
|
||||
model.packed_modules_mapping = {
|
||||
"gate_up_proj": [
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
],
|
||||
}
|
||||
model.embedding_modules = {"lm_head": "lm_head"}
|
||||
model.unpadded_vocab_size = 32064
|
||||
|
||||
return model
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mixtral_lora_files():
|
||||
# Note: this module has incorrect adapter_config.json to test
|
||||
# https://github.com/vllm-project/vllm/pull/5909/files.
|
||||
return hf_api().snapshot_download(repo_id="SangBinCho/mixtral-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def chatglm3_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def baichuan_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def baichuan_zero_lora_files():
|
||||
# all the lora_B weights are initialized to zero.
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def baichuan_regex_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan-7b-lora-zero-regex")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def ilama_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/ilama-text2sql-spider")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def minicpmv_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen2vl_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen25vl_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/qwen25-vl-lora-pokemon")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen2vl_language_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="prashanth058/qwen2vl-flickr-lora-language"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen2vl_vision_tower_connector_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="prashanth058/qwen2vl-flickr-lora-tower-connector"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen2vl_vision_tower_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="prashanth058/qwen2vl-flickr-lora-tower")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen25vl_vision_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="EpochEcho/qwen2.5-3b-vl-lora-vision-connector"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3vl_vision_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="EpochEcho/qwen3-4b-vl-lora-vision-connector"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gemma4_vision_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_meowing_lora_files():
|
||||
"""Download Qwen3 Meow LoRA files once per test session."""
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_woofing_lora_files():
|
||||
"""Download Qwen3 Woof LoRA files once per test session."""
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_guard_star_trek_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="geoffmunn/Qwen3Guard-StarTrek-Classification-0.6B",
|
||||
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_guard_new_zealand_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="geoffmunn/Qwen3Guard-NewZealand-Classification-0.6B",
|
||||
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def skywork_qwen3_reward_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="AmirMohseni/skywork-qwen3-0.6b-reward-lora",
|
||||
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def deepseekv2_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def gptoss20b_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="jeeejeee/gpt-oss-20b-lora-adapter-text2sql"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3moe_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/qwen3-moe-text2sql-spider")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def olmoe_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/olmoe-instruct-text2sql-spider")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llama32_lora_huggingface_id():
|
||||
# huggingface repo id is used to test lora runtime downloading.
|
||||
return "jeeejeee/llama32-3b-text2sql-spider"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llama32_lora_files(llama32_lora_huggingface_id):
|
||||
return hf_api().snapshot_download(repo_id=llama32_lora_huggingface_id)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def whisper_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="chengyili2005/whisper-small-mandarin-lora"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen35_text_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jeeejeee/qwen35-4b-text-only-sql-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen35_vl_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen36_moe_2d_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="jeeejeee/qwen36-35ba3b-2d-weights-poken-lora"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen36_moe_3d_lora_files():
|
||||
return hf_api().snapshot_download(
|
||||
repo_id="jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_default_device():
|
||||
"""Some tests, such as `test_punica_ops.py`, explicitly set the
|
||||
default device, which can affect subsequent tests. Adding this fixture
|
||||
helps avoid this problem.
|
||||
"""
|
||||
original_device = torch.get_default_device()
|
||||
yield
|
||||
torch.set_default_device(original_device)
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.lora.layers.utils import _get_lora_device
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def _param() -> nn.Parameter:
|
||||
return nn.Parameter(torch.empty(1), requires_grad=False)
|
||||
|
||||
|
||||
def test_get_lora_device_unquantized():
|
||||
base_layer = nn.Module()
|
||||
base_layer.weight = _param()
|
||||
assert _get_lora_device(base_layer) == base_layer.weight.device
|
||||
|
||||
|
||||
def test_get_lora_device_gptq_awq():
|
||||
base_layer = nn.Module()
|
||||
base_layer.qweight = _param()
|
||||
assert _get_lora_device(base_layer) == base_layer.qweight.device
|
||||
|
||||
|
||||
def test_get_lora_device_ark_linear():
|
||||
base_layer = nn.Module()
|
||||
base_layer.ark_linear = nn.Module()
|
||||
base_layer.ark_linear.qweight = _param()
|
||||
assert _get_lora_device(base_layer) == base_layer.ark_linear.qweight.device
|
||||
|
||||
|
||||
def test_get_lora_device_unsupported_raises():
|
||||
base_layer = nn.Module()
|
||||
with pytest.raises(ValueError, match="Unsupported base layer"):
|
||||
_get_lora_device(base_layer)
|
||||
@@ -1,114 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Script to test add_lora, remove_lora, pin_lora, list_loras functions."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_async_engine_client_from_engine_args,
|
||||
)
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
MODEL_PATH = "Qwen/Qwen3-0.6B"
|
||||
LORA_MODULE_PATH = "charent/self_cognition_Alice"
|
||||
LORA_RANK = 8
|
||||
|
||||
|
||||
def make_lora_request(lora_id: int):
|
||||
return LoRARequest(
|
||||
lora_name=f"{lora_id}", lora_int_id=lora_id, lora_path=LORA_MODULE_PATH
|
||||
)
|
||||
|
||||
|
||||
def test_lora_functions_sync():
|
||||
max_loras = 4
|
||||
# Create engine in eager-mode. Due to high max_loras, the CI can
|
||||
# OOM during cuda-graph capture.
|
||||
engine_args = EngineArgs(
|
||||
model=MODEL_PATH,
|
||||
enable_lora=True,
|
||||
max_loras=max_loras,
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=128,
|
||||
gpu_memory_utilization=0.8,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
llm = LLMEngine.from_engine_args(engine_args)
|
||||
|
||||
def run_check(fn, args, expected: list):
|
||||
fn(args)
|
||||
assert set(llm.list_loras()) == set(expected)
|
||||
|
||||
run_check(llm.add_lora, make_lora_request(1), [1])
|
||||
run_check(llm.add_lora, make_lora_request(2), [1, 2])
|
||||
|
||||
# Pin LoRA 1 and test that it is never removed on subsequent adds.
|
||||
run_check(llm.pin_lora, 1, [1, 2])
|
||||
run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
|
||||
run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
|
||||
run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
|
||||
run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
|
||||
run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
|
||||
run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
|
||||
run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
|
||||
run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
|
||||
|
||||
# Remove LoRA 1 and continue adding.
|
||||
run_check(llm.remove_lora, 1, [8, 9, 10])
|
||||
run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
|
||||
run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
|
||||
run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
|
||||
|
||||
# Remove all LoRAs.
|
||||
run_check(llm.remove_lora, 13, [12, 10, 11])
|
||||
run_check(llm.remove_lora, 12, [10, 11])
|
||||
run_check(llm.remove_lora, 11, [10])
|
||||
run_check(llm.remove_lora, 10, [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lora_functions_async():
|
||||
max_loras = 4
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=MODEL_PATH,
|
||||
enable_lora=True,
|
||||
max_loras=max_loras,
|
||||
max_lora_rank=LORA_RANK,
|
||||
max_model_len=128,
|
||||
gpu_memory_utilization=0.8,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
async def run_check(fn, args, expected: list):
|
||||
await fn(args)
|
||||
assert set(await llm.list_loras()) == set(expected)
|
||||
|
||||
async with build_async_engine_client_from_engine_args(engine_args) as llm:
|
||||
await run_check(llm.add_lora, make_lora_request(1), [1])
|
||||
await run_check(llm.add_lora, make_lora_request(2), [1, 2])
|
||||
|
||||
# Pin LoRA 1 and test that it is never removed on subsequent adds.
|
||||
await run_check(llm.pin_lora, 1, [1, 2])
|
||||
await run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
|
||||
await run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
|
||||
await run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
|
||||
await run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
|
||||
await run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
|
||||
await run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
|
||||
await run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
|
||||
await run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
|
||||
|
||||
# Remove LoRA 1 and continue adding.
|
||||
await run_check(llm.remove_lora, 1, [8, 9, 10])
|
||||
await run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
|
||||
await run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
|
||||
await run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
|
||||
|
||||
# Remove all LoRAs
|
||||
await run_check(llm.remove_lora, 13, [12, 10, 11])
|
||||
await run_check(llm.remove_lora, 12, [10, 11])
|
||||
await run_check(llm.remove_lora, 11, [10])
|
||||
await run_check(llm.remove_lora, 10, [])
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.lora.utils import is_in_target_modules
|
||||
|
||||
|
||||
class TestIsInTargetModules:
|
||||
"""Tests for is_in_target_modules (deployment-time filter)."""
|
||||
|
||||
def test_none_allows_all(self):
|
||||
assert is_in_target_modules("model.layers.0.self_attn.o_proj", None)
|
||||
|
||||
def test_suffix_in_target(self):
|
||||
assert is_in_target_modules(
|
||||
"model.layers.0.self_attn.o_proj", ["o_proj", "q_proj"]
|
||||
)
|
||||
|
||||
def test_suffix_not_in_target(self):
|
||||
assert not is_in_target_modules(
|
||||
"model.layers.0.self_attn.o_proj", ["q_proj", "k_proj"]
|
||||
)
|
||||
|
||||
def test_empty_target_modules(self):
|
||||
assert not is_in_target_modules("model.layers.0.self_attn.o_proj", [])
|
||||
|
||||
def test_exact_name_match(self):
|
||||
assert is_in_target_modules("dense1", ["dense1", "dense2"])
|
||||
|
||||
def test_exact_name_no_match(self):
|
||||
assert not is_in_target_modules("dense3", ["dense1", "dense2"])
|
||||
|
||||
def test_packed_parent_matches_child_target_modules(self):
|
||||
assert is_in_target_modules(
|
||||
"model.layers.0.mlp.gate_up_proj",
|
||||
["gate_proj", "up_proj"],
|
||||
{"gate_up_proj": ["gate_proj", "up_proj"]},
|
||||
)
|
||||
|
||||
def test_packed_child_matches_parent_target_modules(self):
|
||||
assert is_in_target_modules(
|
||||
"model.layers.0.mlp.gate_proj",
|
||||
["gate_up_proj"],
|
||||
{"gate_up_proj": ["gate_proj", "up_proj"]},
|
||||
)
|
||||
|
||||
def test_fused_parent_matches_child_target_modules(self):
|
||||
assert is_in_target_modules(
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj",
|
||||
["q_a_proj", "kv_a_proj_with_mqa"],
|
||||
{"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"]},
|
||||
)
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
RANK, ALPHA = 16, 32
|
||||
# PEFTHelper stores alpha/sqrt(r) for use_rslora adapters.
|
||||
RSLORA_SCALING = ALPHA / RANK**0.5
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def should_do_global_cleanup_after_test():
|
||||
# Pure-python weight packing; skip GPU/ray cleanup.
|
||||
return False
|
||||
|
||||
|
||||
def moe_loras(scaling: float, stacked: bool = False) -> list[LoRALayerWeights]:
|
||||
a_shape = (3, RANK, 4) if stacked else (RANK, 4)
|
||||
b_shape = (3, 6, RANK) if stacked else (6, RANK)
|
||||
return [
|
||||
LoRALayerWeights(
|
||||
module_name=f"experts.w{i}",
|
||||
rank=RANK,
|
||||
lora_alpha=ALPHA,
|
||||
lora_a=torch.randn(*a_shape),
|
||||
lora_b=torch.randn(*b_shape),
|
||||
scaling=scaling,
|
||||
)
|
||||
for i in (1, 2, 3)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"packer,stacked",
|
||||
[
|
||||
(PackedLoRALayerWeights.pack_moe, False),
|
||||
(PackedLoRALayerWeights.pack_moe_stacked, True),
|
||||
],
|
||||
)
|
||||
def test_moe_packing_uses_stored_scaling(packer, stacked: bool):
|
||||
packed = packer(moe_loras(RSLORA_SCALING, stacked), "experts")
|
||||
assert packed.scaling == pytest.approx([RSLORA_SCALING] * 3)
|
||||
|
||||
|
||||
def test_non_gated_moe_keeps_w3_unscaled():
|
||||
packed = PackedLoRALayerWeights.pack_moe(
|
||||
moe_loras(RSLORA_SCALING), "experts", is_non_gated_moe=True
|
||||
)
|
||||
assert packed.scaling[:2] == pytest.approx([RSLORA_SCALING] * 2)
|
||||
assert packed.scaling[2] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"packer,stacked",
|
||||
[
|
||||
(PackedLoRALayerWeights.pack_moe, False),
|
||||
(PackedLoRALayerWeights.pack_moe_stacked, True),
|
||||
],
|
||||
)
|
||||
def test_moe_packing_keeps_per_projection_scaling(packer, stacked: bool):
|
||||
# e.g. rank_pattern / alpha_pattern giving gate, down and up different scalings
|
||||
loras = moe_loras(1.0, stacked)
|
||||
for lora, scaling in zip(loras, (2.0, 4.0, 8.0)):
|
||||
lora.scaling = scaling
|
||||
packed = packer(loras, "experts")
|
||||
assert packed.scaling == pytest.approx([2.0, 4.0, 8.0])
|
||||
|
||||
|
||||
def test_moe_packing_keeps_per_expert_scaling():
|
||||
"""Experts of one projection can have different scaling, e.g. from
|
||||
alpha_pattern."""
|
||||
loras = moe_loras(RSLORA_SCALING) + moe_loras(RSLORA_SCALING)
|
||||
loras[4].scaling = 2 * RSLORA_SCALING # w2 of expert 1
|
||||
expected = [lora.lora_b * lora.scaling for lora in loras]
|
||||
packed = PackedLoRALayerWeights.pack_moe(loras, "experts").optimize()
|
||||
for i, lora_b in enumerate(packed.lora_b):
|
||||
torch.testing.assert_close(lora_b, torch.stack(expected[i::3]))
|
||||
|
||||
|
||||
def test_moe_packing_rejects_per_expert_rank():
|
||||
"""Experts of one projection must share a rank to be stacked."""
|
||||
loras = moe_loras(1.0) + moe_loras(1.0)
|
||||
loras[4].lora_a = torch.randn(RANK // 2, 4)
|
||||
loras[4].lora_b = torch.randn(6, RANK // 2)
|
||||
with pytest.raises(ValueError, match="different LoRA ranks"):
|
||||
PackedLoRALayerWeights.pack_moe(loras, "experts")
|
||||
|
||||
|
||||
def test_from_config_uses_checkpoint_module_name():
|
||||
peft_helper = PEFTHelper(
|
||||
r=RANK,
|
||||
lora_alpha=ALPHA,
|
||||
target_modules=["q_proj"],
|
||||
rank_pattern={r"^model\.layers\.0\.self_attn\.q_proj": 4},
|
||||
)
|
||||
# the pattern refers to the checkpoint name, not the name after weights mapping
|
||||
lora = LoRALayerWeights.from_config(
|
||||
"language_model.model.layers.0.self_attn.q_proj",
|
||||
peft_helper,
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
)
|
||||
assert lora.rank == 4
|
||||
assert lora.scaling == pytest.approx(ALPHA / 4)
|
||||
lora = LoRALayerWeights.from_config("model.layers.1.self_attn.q_proj", peft_helper)
|
||||
assert lora.rank == RANK
|
||||
assert lora.scaling == pytest.approx(ALPHA / RANK)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end correctness test for 2D MoE LoRA expert-parallel
|
||||
load-time slicing
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
NUM_LAYERS = 48
|
||||
GLOBAL_NUM_EXPERTS = 128
|
||||
LOCAL_NUM_EXPERTS = 64 # ep_size = 2
|
||||
EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
|
||||
NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
|
||||
|
||||
|
||||
def _expected_lora_modules() -> set[str]:
|
||||
"""Replicate the set ``WorkerLoRAManager._load_adapter`` would build
|
||||
from this model's ``packed_modules_mapping``."""
|
||||
expected: set[str] = set(NON_EXPERT_MODULES)
|
||||
for expert in range(GLOBAL_NUM_EXPERTS):
|
||||
for proj in EXPERT_PROJECTIONS:
|
||||
expected.add(f"experts.{expert}.{proj}")
|
||||
return expected
|
||||
|
||||
|
||||
def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
|
||||
return LoRAModel.from_local_checkpoint(
|
||||
lora_dir,
|
||||
_expected_lora_modules(),
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=lora_id,
|
||||
device="cpu",
|
||||
moe_ep_spec=moe_ep_spec,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_rank", [0, 1])
|
||||
def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
|
||||
"""ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
|
||||
LoRA has the right size, the right expert membership, and the
|
||||
right tensor values."""
|
||||
peft_helper = PEFTHelper.from_local_dir(
|
||||
qwen3moe_lora_files, max_position_embeddings=4096
|
||||
)
|
||||
|
||||
# Baseline: no spec → loads every expert × projection × layer plus
|
||||
# all non-expert LoRA modules.
|
||||
ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
|
||||
expected_baseline = (
|
||||
GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
|
||||
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
|
||||
)
|
||||
assert len(ground_truth.loras) == expected_baseline
|
||||
|
||||
# Sliced load: only this rank's experts; non-expert LoRA is untouched.
|
||||
spec = MoEEPLoadSpec(
|
||||
ep_rank=ep_rank,
|
||||
local_num_experts=LOCAL_NUM_EXPERTS,
|
||||
global_num_experts=GLOBAL_NUM_EXPERTS,
|
||||
)
|
||||
sliced = _load(
|
||||
qwen3moe_lora_files,
|
||||
peft_helper,
|
||||
moe_ep_spec=spec,
|
||||
lora_id=100 + ep_rank,
|
||||
)
|
||||
|
||||
expected_sliced = (
|
||||
LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
|
||||
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
|
||||
)
|
||||
assert len(sliced.loras) == expected_sliced
|
||||
|
||||
expert_start = ep_rank * LOCAL_NUM_EXPERTS
|
||||
expert_end = expert_start + LOCAL_NUM_EXPERTS
|
||||
|
||||
for name, lora in sliced.loras.items():
|
||||
gt = ground_truth.loras[name]
|
||||
torch.testing.assert_close(lora.lora_a, gt.lora_a)
|
||||
torch.testing.assert_close(lora.lora_b, gt.lora_b)
|
||||
if ".experts." in name:
|
||||
expert_idx = int(name.split(".experts.")[-1].split(".")[0])
|
||||
assert expert_start <= expert_idx < expert_end, (
|
||||
f"non-local expert {expert_idx} leaked: {name}"
|
||||
)
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
ERROR_CASES = [
|
||||
(
|
||||
"test_rank",
|
||||
{"r": 1024},
|
||||
"is greater than max_lora_rank",
|
||||
),
|
||||
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
|
||||
(
|
||||
"test_modules_to_save",
|
||||
{"modules_to_save": ["lm_head"]},
|
||||
"Unsupported modules_to_save",
|
||||
),
|
||||
(
|
||||
"test_rank_pattern",
|
||||
{"r": 8, "rank_pattern": {"q_proj": 32}},
|
||||
"LoRA rank 32 is greater than max_lora_rank",
|
||||
),
|
||||
("test_rank_zero", {"r": 0}, "must be a positive integer"),
|
||||
("test_rank_negative", {"r": -8}, "must be a positive integer"),
|
||||
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
|
||||
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
|
||||
(
|
||||
"test_pissa_niter",
|
||||
{"init_lora_weights": "pissa_niter_4"},
|
||||
"init_lora_weights='pissa_niter_4'",
|
||||
),
|
||||
("test_olora", {"init_lora_weights": "olora"}, "init_lora_weights='olora'"),
|
||||
("test_corda", {"init_lora_weights": "corda"}, "init_lora_weights='corda'"),
|
||||
("test_loftq", {"init_lora_weights": "loftq"}, "init_lora_weights='loftq'"),
|
||||
(
|
||||
"test_unknown_init",
|
||||
{"init_lora_weights": "future_init"},
|
||||
"init_lora_weights='future_init'",
|
||||
),
|
||||
("test_alora", {"alora_invocation_tokens": [1, 2]}, "Activated LoRA"),
|
||||
("test_layer_replication", {"layer_replication": [[0, 2]]}, "layer_replication"),
|
||||
("test_bdlora", {"use_bdlora": {"nblocks": 2}}, "BD-LoRA"),
|
||||
("test_qalora", {"use_qalora": True}, "QALoRA"),
|
||||
]
|
||||
|
||||
|
||||
def test_peft_helper_pass(llama32_lora_files, tmp_path):
|
||||
peft_helper = PEFTHelper.from_local_dir(
|
||||
llama32_lora_files, max_position_embeddings=4096
|
||||
)
|
||||
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
|
||||
peft_helper.validate_legal(lora_config)
|
||||
assert peft_helper.r == 8
|
||||
assert peft_helper.lora_alpha == 32
|
||||
target_modules = sorted(peft_helper.target_modules)
|
||||
|
||||
assert target_modules == [
|
||||
"down_proj",
|
||||
"embed_tokens",
|
||||
"gate_proj",
|
||||
"k_proj",
|
||||
"lm_head",
|
||||
"o_proj",
|
||||
"q_proj",
|
||||
"up_proj",
|
||||
"v_proj",
|
||||
]
|
||||
assert peft_helper.vllm_max_position_embeddings == 4096
|
||||
|
||||
# test RSLoRA
|
||||
rslora_config = dict(use_rslora=True)
|
||||
test_dir = tmp_path / "test_rslora"
|
||||
shutil.copytree(llama32_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
with open(config_path) as f:
|
||||
adapter_config = json.load(f)
|
||||
# Apply configuration changes
|
||||
adapter_config.update(rslora_config)
|
||||
|
||||
# Save modified configuration
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(adapter_config, f)
|
||||
|
||||
peft_helper = PEFTHelper.from_local_dir(test_dir, max_position_embeddings=4096)
|
||||
peft_helper.validate_legal(lora_config)
|
||||
scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
|
||||
assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
|
||||
def test_peft_helper_error(
|
||||
llama32_lora_files,
|
||||
tmp_path,
|
||||
test_name: str,
|
||||
config_change: dict,
|
||||
expected_error: str,
|
||||
):
|
||||
test_dir = tmp_path / test_name
|
||||
shutil.copytree(llama32_lora_files, test_dir)
|
||||
|
||||
# Load and modify configuration
|
||||
config_path = test_dir / "adapter_config.json"
|
||||
with open(config_path) as f:
|
||||
adapter_config = json.load(f)
|
||||
# Apply configuration changes
|
||||
adapter_config.update(config_change)
|
||||
|
||||
# Save modified configuration
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(adapter_config, f)
|
||||
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
|
||||
# Test loading the adapter
|
||||
with pytest.raises(ValueError, match=expected_error):
|
||||
PEFTHelper.from_local_dir(
|
||||
test_dir, max_position_embeddings=4096
|
||||
).validate_legal(lora_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_rank", [0, -1, -8])
|
||||
def test_peft_helper_invalid_rank_direct(bad_rank: int):
|
||||
"""Regression test: constructing a PEFTHelper with a non-positive rank
|
||||
must raise a clear ValueError instead of crashing with an unrelated
|
||||
ZeroDivisionError (r=0) or silently succeeding with a sign-flipped
|
||||
scaling factor that validate_legal() never catches (r<0, since its only
|
||||
rank check is the upper bound against max_lora_rank).
|
||||
|
||||
Network-free: constructs PEFTHelper directly rather than going through
|
||||
from_local_dir(), which needs an on-disk adapter_config.json.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="must be a positive integer"):
|
||||
PEFTHelper(r=bad_rank, lora_alpha=16, target_modules=["q_proj"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"init_lora_weights",
|
||||
[True, False, "gaussian", "eva", "orthogonal", "mica", "lora_ga"],
|
||||
)
|
||||
def test_peft_helper_init_lora_weights_supported(init_lora_weights):
|
||||
"""A saved adapter with these inits loads in PEFT as a plain LoRA."""
|
||||
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
|
||||
PEFTHelper.from_dict(
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 16,
|
||||
"target_modules": ["q_proj"],
|
||||
"init_lora_weights": init_lora_weights,
|
||||
"alora_invocation_tokens": None,
|
||||
"layer_replication": None,
|
||||
"use_bdlora": None,
|
||||
}
|
||||
).validate_legal(lora_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config,module_name,expected",
|
||||
[
|
||||
# rank_pattern only: alpha / r_m
|
||||
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.q_proj", (4, 8.0)),
|
||||
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.v_proj", (16, 2.0)),
|
||||
# alpha_pattern only: alpha_m / r
|
||||
(
|
||||
{"alpha_pattern": {"v_proj": 64}},
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
(16, 4.0),
|
||||
),
|
||||
# keys only match at a "." boundary, like in PEFT
|
||||
({"rank_pattern": {"proj": 4}}, "model.layers.0.self_attn.q_proj", (16, 2.0)),
|
||||
# fused MoE experts from PEFT `target_parameters`: gate_up_proj is stored
|
||||
# as `experts.base_layer`, down_proj as `experts`
|
||||
(
|
||||
{"rank_pattern": {"experts.gate_up_proj": 2}},
|
||||
"model.layers.0.mlp.experts.base_layer",
|
||||
(2, 16.0),
|
||||
),
|
||||
(
|
||||
{"rank_pattern": {"experts.down_proj": 2}},
|
||||
"model.layers.0.mlp.experts",
|
||||
(2, 16.0),
|
||||
),
|
||||
(
|
||||
{"rank_pattern": {"experts.gate_up_proj": 2}},
|
||||
"model.layers.0.mlp.experts",
|
||||
(16, 2.0),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_peft_helper_rank_and_alpha_pattern(config, module_name, expected):
|
||||
peft_helper = PEFTHelper(
|
||||
r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], **config
|
||||
)
|
||||
rank, scaling = peft_helper.get_rank_and_scaling(module_name)
|
||||
assert rank == expected[0]
|
||||
assert scaling == pytest.approx(expected[1])
|
||||
|
||||
|
||||
def test_peft_helper_pattern_rslora():
|
||||
peft_helper = PEFTHelper(
|
||||
r=16,
|
||||
lora_alpha=32,
|
||||
target_modules=["q_proj"],
|
||||
use_rslora=True,
|
||||
rank_pattern={"q_proj": 4},
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.q_proj") == (4, 32 / math.sqrt(4))
|
||||
|
||||
|
||||
def test_peft_helper_dynamic_rank_conversion():
|
||||
"""PEFT's `save_as_lora` with a dynamic rank writes r=1, lora_alpha=1 and the
|
||||
same per-module values in rank_pattern and alpha_pattern, so every module
|
||||
keeps a scaling of 1."""
|
||||
peft_helper = PEFTHelper(
|
||||
r=1,
|
||||
lora_alpha=1,
|
||||
target_modules=["q_proj", "v_proj"],
|
||||
rank_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
|
||||
alpha_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.q_proj") == (
|
||||
32,
|
||||
1.0,
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.v_proj") == (
|
||||
3,
|
||||
1.0,
|
||||
)
|
||||
with pytest.raises(ValueError, match="LoRA rank 32 is greater than max_lora_rank"):
|
||||
peft_helper.validate_legal(LoRAConfig(max_lora_rank=16))
|
||||
|
||||
|
||||
def test_peft_helper_null_patterns():
|
||||
peft_helper = PEFTHelper.from_dict(
|
||||
{
|
||||
"r": 8,
|
||||
"lora_alpha": 16,
|
||||
"target_modules": ["q_proj"],
|
||||
"rank_pattern": None,
|
||||
"alpha_pattern": None,
|
||||
}
|
||||
)
|
||||
assert peft_helper.get_rank_and_scaling("model.q_proj") == (8, 2.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_rank", [0, -4, 2.5])
|
||||
def test_peft_helper_invalid_rank_pattern(bad_rank):
|
||||
with pytest.raises(ValueError, match="rank_pattern"):
|
||||
PEFTHelper(
|
||||
r=8,
|
||||
lora_alpha=16,
|
||||
target_modules=["q_proj"],
|
||||
rank_pattern={"q_proj": bad_rank},
|
||||
)
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
|
||||
|
||||
class DummyLoRAResolver(LoRAResolver):
|
||||
"""A dummy LoRA resolver for testing."""
|
||||
|
||||
async def resolve_lora(
|
||||
self, base_model_name: str, lora_name: str
|
||||
) -> LoRARequest | None:
|
||||
if lora_name == "test_lora":
|
||||
return LoRARequest(
|
||||
lora_name=lora_name,
|
||||
lora_path=f"/dummy/path/{base_model_name}/{lora_name}",
|
||||
lora_int_id=abs(hash(lora_name)),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def test_resolver_registry_registration():
|
||||
"""Test basic resolver registration functionality."""
|
||||
registry = LoRAResolverRegistry
|
||||
resolver = DummyLoRAResolver()
|
||||
|
||||
# Register a new resolver
|
||||
registry.register_resolver("dummy", resolver)
|
||||
assert "dummy" in registry.get_supported_resolvers()
|
||||
|
||||
# Get registered resolver
|
||||
retrieved_resolver = registry.get_resolver("dummy")
|
||||
assert retrieved_resolver is resolver
|
||||
|
||||
|
||||
def test_resolver_registry_duplicate_registration():
|
||||
"""Test registering a resolver with an existing name."""
|
||||
registry = LoRAResolverRegistry
|
||||
resolver1 = DummyLoRAResolver()
|
||||
resolver2 = DummyLoRAResolver()
|
||||
|
||||
registry.register_resolver("dummy", resolver1)
|
||||
registry.register_resolver("dummy", resolver2)
|
||||
|
||||
assert registry.get_resolver("dummy") is resolver2
|
||||
|
||||
|
||||
def test_resolver_registry_unknown_resolver():
|
||||
"""Test getting a non-existent resolver."""
|
||||
registry = LoRAResolverRegistry
|
||||
|
||||
with pytest.raises(KeyError, match="not found"):
|
||||
registry.get_resolver("unknown_resolver")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dummy_resolver_resolve():
|
||||
"""Test the dummy resolver's resolve functionality."""
|
||||
dummy_resolver = DummyLoRAResolver()
|
||||
base_model_name = "base_model_test"
|
||||
lora_name = "test_lora"
|
||||
|
||||
# Test successful resolution
|
||||
result = await dummy_resolver.resolve_lora(base_model_name, lora_name)
|
||||
assert isinstance(result, LoRARequest)
|
||||
assert result.lora_name == lora_name
|
||||
assert result.lora_path == f"/dummy/path/{base_model_name}/{lora_name}"
|
||||
|
||||
# Test failed resolution
|
||||
result = await dummy_resolver.resolve_lora(base_model_name, "nonexistent_lora")
|
||||
assert result is None
|
||||
@@ -1,203 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from huggingface_hub.utils import HfHubHTTPError
|
||||
from torch import nn
|
||||
|
||||
from vllm.lora.utils import (
|
||||
get_adapter_absolute_path,
|
||||
parse_fine_tuned_lora_name,
|
||||
replace_submodule,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
class LoRANameParserTestConfig(NamedTuple):
|
||||
name: str
|
||||
module_name: str
|
||||
is_lora_a: bool
|
||||
weights_mapper: WeightsMapper | None = None
|
||||
|
||||
|
||||
def test_parse_fine_tuned_lora_name_valid():
|
||||
fixture = [
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.lm_head.lora_A.weight", "lm_head", True, False
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.lm_head.lora_B.weight", "lm_head", False, False
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.embed_tokens.lora_embedding_A",
|
||||
"model.embed_tokens",
|
||||
True,
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.embed_tokens.lora_embedding_B",
|
||||
"model.embed_tokens",
|
||||
False,
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
|
||||
"model.layers.9.mlp.down_proj",
|
||||
True,
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
|
||||
"model.layers.9.mlp.down_proj",
|
||||
False,
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"language_model.layers.9.mlp.down_proj.lora_A.weight",
|
||||
"language_model.layers.9.mlp.down_proj",
|
||||
True,
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"language_model.layers.9.mlp.down_proj.lora_B.weight",
|
||||
"language_model.layers.9.mlp.down_proj",
|
||||
False,
|
||||
),
|
||||
# Test with WeightsMapper
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
|
||||
"language_model.model.layers.9.mlp.down_proj",
|
||||
True,
|
||||
weights_mapper=WeightsMapper(
|
||||
orig_to_new_prefix={"model.": "language_model.model."}
|
||||
),
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
|
||||
"language_model.model.layers.9.mlp.down_proj",
|
||||
False,
|
||||
weights_mapper=WeightsMapper(
|
||||
orig_to_new_prefix={"model.": "language_model.model."}
|
||||
),
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"model.layers.9.mlp.down_proj.lora_A.weight",
|
||||
"language_model.model.layers.9.mlp.down_proj",
|
||||
True,
|
||||
weights_mapper=WeightsMapper(
|
||||
orig_to_new_prefix={"model.": "language_model.model."}
|
||||
),
|
||||
),
|
||||
LoRANameParserTestConfig(
|
||||
"model.layers.9.mlp.down_proj.lora_B.weight",
|
||||
"language_model.model.layers.9.mlp.down_proj",
|
||||
False,
|
||||
weights_mapper=WeightsMapper(
|
||||
orig_to_new_prefix={"model.": "language_model.model."}
|
||||
),
|
||||
),
|
||||
]
|
||||
for name, module_name, is_lora_a, weights_mapper in fixture:
|
||||
assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
|
||||
name, weights_mapper
|
||||
)
|
||||
|
||||
|
||||
def test_parse_fine_tuned_lora_name_invalid():
|
||||
fixture = {
|
||||
"base_model.weight",
|
||||
"base_model.model.weight",
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_A.bias",
|
||||
"base_model.model.model.layers.9.mlp.down_proj.lora_B.bias",
|
||||
}
|
||||
for name in fixture:
|
||||
with pytest.raises(ValueError, match="unsupported LoRA weight"):
|
||||
parse_fine_tuned_lora_name(name)
|
||||
|
||||
|
||||
def test_replace_submodule():
|
||||
model = nn.Sequential(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", nn.Linear(764, 100)),
|
||||
("act1", nn.ReLU()),
|
||||
("dense2", nn.Linear(100, 50)),
|
||||
(
|
||||
"seq1",
|
||||
nn.Sequential(
|
||||
OrderedDict(
|
||||
[
|
||||
("dense1", nn.Linear(100, 10)),
|
||||
("dense2", nn.Linear(10, 50)),
|
||||
]
|
||||
)
|
||||
),
|
||||
),
|
||||
("act2", nn.ReLU()),
|
||||
("output", nn.Linear(50, 10)),
|
||||
("outact", nn.Sigmoid()),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
sigmoid = nn.Sigmoid()
|
||||
|
||||
replace_submodule(model, "act1", sigmoid)
|
||||
assert dict(model.named_modules())["act1"] == sigmoid
|
||||
|
||||
dense2 = nn.Linear(1, 5)
|
||||
replace_submodule(model, "seq1.dense2", dense2)
|
||||
assert dict(model.named_modules())["seq1.dense2"] == dense2
|
||||
|
||||
|
||||
# Unit tests for get_adapter_absolute_path
|
||||
@patch("os.path.isabs")
|
||||
def test_get_adapter_absolute_path_absolute(mock_isabs):
|
||||
path = "/absolute/path/to/lora"
|
||||
mock_isabs.return_value = True
|
||||
assert get_adapter_absolute_path(path) == path
|
||||
|
||||
|
||||
@patch("os.path.expanduser")
|
||||
def test_get_adapter_absolute_path_expanduser(mock_expanduser):
|
||||
# Path with ~ that needs to be expanded
|
||||
path = "~/relative/path/to/lora"
|
||||
absolute_path = "/home/user/relative/path/to/lora"
|
||||
mock_expanduser.return_value = absolute_path
|
||||
assert get_adapter_absolute_path(path) == absolute_path
|
||||
|
||||
|
||||
@patch("os.path.exists")
|
||||
@patch("os.path.abspath")
|
||||
def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
|
||||
# Relative path that exists locally
|
||||
path = "relative/path/to/lora"
|
||||
absolute_path = "/absolute/path/to/lora"
|
||||
mock_exist.return_value = True
|
||||
mock_abspath.return_value = absolute_path
|
||||
assert get_adapter_absolute_path(path) == absolute_path
|
||||
|
||||
|
||||
@patch("huggingface_hub.HfApi.snapshot_download")
|
||||
@patch("os.path.exists")
|
||||
def test_get_adapter_absolute_path_huggingface(mock_exist, mock_snapshot_download):
|
||||
# Hugging Face model identifier
|
||||
path = "org/repo"
|
||||
absolute_path = "/mock/snapshot/path"
|
||||
mock_exist.return_value = False
|
||||
mock_snapshot_download.return_value = absolute_path
|
||||
assert get_adapter_absolute_path(path) == absolute_path
|
||||
|
||||
|
||||
@patch("huggingface_hub.HfApi.snapshot_download")
|
||||
@patch("os.path.exists")
|
||||
def test_get_adapter_absolute_path_huggingface_error(
|
||||
mock_exist, mock_snapshot_download
|
||||
):
|
||||
# Hugging Face model identifier with download error
|
||||
path = "org/repo"
|
||||
mock_exist.return_value = False
|
||||
mock_snapshot_download.side_effect = HfHubHTTPError(
|
||||
"failed to query model info",
|
||||
response=MagicMock(),
|
||||
)
|
||||
assert get_adapter_absolute_path(path) == path
|
||||
@@ -19,7 +19,6 @@ def test_qwen3_5_lm_head_receives_quant_config():
|
||||
mock_vllm_config.cache_config.mamba_cache_mode = "align"
|
||||
mock_vllm_config.scheduler_config = Mock()
|
||||
mock_vllm_config.quant_config = mock_quant_config
|
||||
mock_vllm_config.lora_config = None
|
||||
|
||||
mock_pp_group = Mock()
|
||||
mock_pp_group.is_last_rank = True
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,6 @@ from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
@@ -26,7 +25,6 @@ AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
"<|end_of_text|>\n"
|
||||
"<|start_of_role|>assistant<|end_of_role|>"
|
||||
),
|
||||
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
|
||||
},
|
||||
"nvidia/audio-flamingo-3-hf": {
|
||||
"prompt": (
|
||||
@@ -90,7 +88,6 @@ def test_transformers_audio_generation(
|
||||
model_id: str,
|
||||
):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
audio_lora_path = settings.get("audio_lora_path")
|
||||
|
||||
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
|
||||
target_sr = settings.get("sampling_rate", orig_sr)
|
||||
@@ -105,19 +102,13 @@ def test_transformers_audio_generation(
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
enable_lora=audio_lora_path is not None,
|
||||
max_lora_rank=64,
|
||||
**settings.get("vllm_runner_kwargs", {}),
|
||||
) as vllm_model:
|
||||
lora_request = (
|
||||
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
|
||||
)
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[settings["prompt"]],
|
||||
128,
|
||||
num_logprobs=10,
|
||||
audios=[audio],
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name="vllm_add_dummy_endpoint_plugin",
|
||||
version="0.1",
|
||||
packages=["vllm_add_dummy_endpoint_plugin"],
|
||||
entry_points={
|
||||
"vllm.endpoint_plugins": [
|
||||
"dummy_admin_endpoint_plugin = vllm_add_dummy_endpoint_plugin:DummyAdminEndpointPlugin" # noqa
|
||||
]
|
||||
},
|
||||
)
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worked example `vllm.endpoint_plugins` entry point.
|
||||
|
||||
Reports scheduler config via `collective_rpc`. Demonstrates the full
|
||||
contract: `attach_router` registers the route at Phase A (`build_app`) and
|
||||
`init_state` stashes the `EngineClient` the route handler needs at Phase B
|
||||
(`init_app_state`).
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
|
||||
|
||||
class DummyAdminEndpointPlugin:
|
||||
name = "dummy_admin_endpoint_plugin"
|
||||
required_tasks: tuple[str, ...] | None = None
|
||||
|
||||
def attach_router(self, app: FastAPI) -> None:
|
||||
@app.get("/v1/admin/scheduler_config")
|
||||
async def scheduler_config(raw_request: Request):
|
||||
engine_client = raw_request.app.state.dummy_engine_client
|
||||
results = await engine_client.collective_rpc("get_scheduler_config")
|
||||
return {"scheduler_config": results}
|
||||
|
||||
async def init_state(self, engine_client, state, args) -> None:
|
||||
state.dummy_engine_client = engine_client
|
||||
@@ -1,130 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E tests for GGUF plugin functionality."""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from ...conftest import VllmRunner
|
||||
from ...models.utils import check_logprobs_close
|
||||
from ...utils import multi_gpu_test
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
|
||||
class GGUFTestConfig(NamedTuple):
|
||||
original_model: str
|
||||
gguf_model_path: str # Full path to .gguf file
|
||||
|
||||
|
||||
QWEN3_CONFIG = GGUFTestConfig(
|
||||
original_model="Qwen/Qwen3-0.6B",
|
||||
gguf_model_path="unsloth/Qwen3-0.6B-GGUF:Q8_0",
|
||||
)
|
||||
|
||||
|
||||
OLMOE_CONFIG = GGUFTestConfig(
|
||||
original_model="allenai/OLMoE-1B-7B-0125",
|
||||
gguf_model_path="allenai/OLMoE-1B-7B-0125-GGUF:Q6_K",
|
||||
)
|
||||
|
||||
|
||||
MODELS = [
|
||||
QWEN3_CONFIG,
|
||||
OLMOE_CONFIG,
|
||||
]
|
||||
|
||||
|
||||
def check_model_outputs(
|
||||
vllm_runner: type[VllmRunner],
|
||||
prompts: list[str],
|
||||
model: GGUFTestConfig,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tp_size: int,
|
||||
):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model.original_model)
|
||||
if tokenizer.chat_template is not None:
|
||||
messages = [[{"role": "user", "content": prompt}] for prompt in prompts]
|
||||
prompts = tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
|
||||
# Run gguf model.
|
||||
with vllm_runner(
|
||||
model_name=model.gguf_model_path,
|
||||
enforce_eager=True,
|
||||
tokenizer_name=model.original_model,
|
||||
dtype=dtype,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tp_size,
|
||||
) as gguf_model:
|
||||
gguf_outputs = gguf_model.generate_greedy_logprobs(
|
||||
prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Run unquantized model.
|
||||
# Should run with tp=1, otherwise the test will stuck at
|
||||
# nccl initialization.
|
||||
with vllm_runner(
|
||||
model_name=model.original_model,
|
||||
enforce_eager=True, # faster tests
|
||||
dtype=dtype,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=1,
|
||||
) as original_model:
|
||||
original_outputs = original_model.generate_greedy_logprobs(
|
||||
prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=original_outputs,
|
||||
outputs_1_lst=gguf_outputs,
|
||||
name_0="original",
|
||||
name_1="gguf",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [8])
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
def test_models(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: GGUFTestConfig,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
check_model_outputs(
|
||||
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_distributed(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: GGUFTestConfig,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
check_model_outputs(
|
||||
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
|
||||
)
|
||||
@@ -1,167 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import pytest
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
|
||||
from ...conftest import IMAGE_ASSETS, HfRunner, VllmRunner
|
||||
from ...models.utils import check_logprobs_close
|
||||
|
||||
|
||||
class GGUFMMTestConfig(NamedTuple):
|
||||
original_model: str
|
||||
gguf_repo: str
|
||||
gguf_backbone: str
|
||||
gguf_mmproj: str
|
||||
prompt: list[str]
|
||||
image_names: list[str] # Store names, load PIL images at runtime
|
||||
max_model_len: int = 4096
|
||||
marks: list[MarkDecorator] = []
|
||||
mm_processor_kwargs: dict[str, Any] = {}
|
||||
|
||||
@property
|
||||
def gguf_model(self):
|
||||
hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_mmproj)
|
||||
return hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_backbone)
|
||||
|
||||
|
||||
# Common prompts aligned with test_common.py "gemma3" entry format
|
||||
_GEMMA3_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": (
|
||||
"<bos><start_of_turn>user\n"
|
||||
"<start_of_image>What's the content in the center of the image?"
|
||||
"<end_of_turn>\n<start_of_turn>model\n"
|
||||
),
|
||||
"cherry_blossom": (
|
||||
"<bos><start_of_turn>user\n"
|
||||
"<start_of_image>What is the season?"
|
||||
"<end_of_turn>\n<start_of_turn>model\n"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
# Image asset names - load at runtime to avoid pickle issues with subprocess
|
||||
_GEMMA3_IMAGE_NAMES = ["stop_sign", "cherry_blossom"]
|
||||
|
||||
# Regular multimodal (no pan-and-scan) - uses QAT Q4_0 GGUF
|
||||
GEMMA3_CONFIG = GGUFMMTestConfig(
|
||||
original_model="google/gemma-3-4b-it",
|
||||
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
|
||||
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
|
||||
gguf_mmproj="mmproj-model-f16-4B.gguf",
|
||||
prompt=_GEMMA3_PROMPTS,
|
||||
image_names=_GEMMA3_IMAGE_NAMES,
|
||||
max_model_len=4096,
|
||||
mm_processor_kwargs={},
|
||||
)
|
||||
|
||||
# Pan-and-scan multimodal - uses unquantized BF16 GGUF
|
||||
GEMMA3_CONFIG_PAN_AND_SCAN = GGUFMMTestConfig(
|
||||
original_model="google/gemma-3-4b-it",
|
||||
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
|
||||
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
|
||||
gguf_mmproj="mmproj-model-f16-4B.gguf",
|
||||
prompt=_GEMMA3_PROMPTS,
|
||||
image_names=_GEMMA3_IMAGE_NAMES,
|
||||
max_model_len=4096,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
)
|
||||
|
||||
MODELS_TO_TEST = [GEMMA3_CONFIG, GEMMA3_CONFIG_PAN_AND_SCAN]
|
||||
|
||||
|
||||
def run_multimodal_gguf_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: GGUFMMTestConfig,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
):
|
||||
# Load images at runtime (inside subprocess) to avoid pickle issues
|
||||
images = [ImageAsset(name).pil_image for name in model.image_names]
|
||||
size_factors = [0.25, 0.5, 1.0]
|
||||
inputs_per_image = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
)
|
||||
for image, prompt in zip(images, model.prompt)
|
||||
]
|
||||
|
||||
# NOTE: Run vLLM first to avoid CUDA init issues with multiprocessing fork.
|
||||
# Run GGUF model via vLLM.
|
||||
with (
|
||||
set_default_torch_num_threads(1),
|
||||
vllm_runner(
|
||||
model_name=model.gguf_model,
|
||||
enforce_eager=True,
|
||||
tokenizer_name=model.original_model,
|
||||
dtype=dtype,
|
||||
max_model_len=model.max_model_len,
|
||||
mm_processor_kwargs=model.mm_processor_kwargs,
|
||||
) as gguf_model,
|
||||
):
|
||||
gguf_outputs_per_case = [
|
||||
gguf_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
)
|
||||
for prompts, images in inputs_per_image
|
||||
]
|
||||
|
||||
# Then run HfRunner for HuggingFace baseline comparison.
|
||||
with hf_runner(
|
||||
model.original_model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
) as hf_model:
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
)
|
||||
for prompts, images in inputs_per_image
|
||||
]
|
||||
|
||||
for hf_outputs, gguf_outputs in zip(hf_outputs_per_case, gguf_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=gguf_outputs,
|
||||
name_0="hf",
|
||||
name_1="gguf",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS_TO_TEST)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_gemma3_mm_gguf(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: GGUFMMTestConfig,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
run_multimodal_gguf_test(
|
||||
hf_runner, vllm_runner, model, dtype, max_tokens, num_logprobs
|
||||
)
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.plugins.lora_resolvers.filesystem_resolver import FilesystemResolver
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
LORA_NAME = "charent/self_cognition_Alice"
|
||||
PA_NAME = "swapnilbp/llama_tweet_ptune"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def adapter_cache(request, tmpdir_factory):
|
||||
# Create dir that mimics the structure of the adapter cache
|
||||
adapter_cache = tmpdir_factory.mktemp(request.module.__name__) / "adapter_cache"
|
||||
return adapter_cache
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen3_lora_files():
|
||||
return hf_api().snapshot_download(repo_id=LORA_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pa_files():
|
||||
return hf_api().snapshot_download(repo_id=PA_NAME)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filesystem_resolver(adapter_cache, qwen3_lora_files):
|
||||
model_files = adapter_cache / LORA_NAME
|
||||
shutil.copytree(qwen3_lora_files, model_files)
|
||||
|
||||
fs_resolver = FilesystemResolver(adapter_cache)
|
||||
assert fs_resolver is not None
|
||||
|
||||
lora_request = await fs_resolver.resolve_lora(MODEL_NAME, LORA_NAME)
|
||||
assert lora_request is not None
|
||||
assert lora_request.lora_name == LORA_NAME
|
||||
assert lora_request.lora_path == os.path.join(adapter_cache, LORA_NAME)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_adapter(adapter_cache):
|
||||
fs_resolver = FilesystemResolver(adapter_cache)
|
||||
assert fs_resolver is not None
|
||||
|
||||
missing_lora_request = await fs_resolver.resolve_lora(MODEL_NAME, "foobar")
|
||||
assert missing_lora_request is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonlora_adapter(adapter_cache, pa_files):
|
||||
model_files = adapter_cache / PA_NAME
|
||||
shutil.copytree(pa_files, model_files)
|
||||
|
||||
fs_resolver = FilesystemResolver(adapter_cache)
|
||||
assert fs_resolver is not None
|
||||
|
||||
pa_request = await fs_resolver.resolve_lora(MODEL_NAME, PA_NAME)
|
||||
assert pa_request is None
|
||||
@@ -1,108 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from huggingface_hub.constants import HF_HUB_CACHE
|
||||
|
||||
from vllm.plugins.lora_resolvers.hf_hub_resolver import HfHubResolver
|
||||
|
||||
LORA_LIB_MODEL_NAME = "ibm-granite/granite-3.3-8b-instruct"
|
||||
# Repo with multiple LoRAs contained in it
|
||||
LORA_LIB = "ibm-granite/granite-3.3-8b-rag-agent-lib"
|
||||
LORA_NAME = "ibm-granite/granite-3.3-8b-rag-agent-lib/answerability_prediction_lora" # noqa: E501
|
||||
NON_LORA_SUBPATH = "ibm-granite/granite-3.3-8b-rag-agent-lib/README.md"
|
||||
LIB_DOWNLOAD_DIR = os.path.join(
|
||||
HF_HUB_CACHE, "models--ibm-granite--granite-3.3-8b-rag-agent-lib"
|
||||
)
|
||||
INVALID_REPO_NAME = "thisrepodoesnotexist"
|
||||
|
||||
# Repo with only one LoRA in the root dir
|
||||
LORA_REPO_MODEL_NAME = "meta-llama/Llama-2-7b-hf"
|
||||
LORA_REPO = "yard1/llama-2-7b-sql-lora-test"
|
||||
REPO_DOWNLOAD_DIR = os.path.join(
|
||||
HF_HUB_CACHE, "models--yard1--llama-2-7b-sql-lora-test"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hf_resolver_with_direct_path():
|
||||
hf_resolver = HfHubResolver([LORA_REPO])
|
||||
assert hf_resolver is not None
|
||||
|
||||
lora_request = await hf_resolver.resolve_lora(LORA_REPO_MODEL_NAME, LORA_REPO)
|
||||
assert lora_request is not None
|
||||
assert lora_request.lora_name == LORA_REPO
|
||||
assert REPO_DOWNLOAD_DIR in lora_request.lora_path
|
||||
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hf_resolver_with_nested_paths():
|
||||
hf_resolver = HfHubResolver([LORA_LIB])
|
||||
assert hf_resolver is not None
|
||||
|
||||
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
|
||||
assert lora_request is not None
|
||||
assert lora_request.lora_name == LORA_NAME
|
||||
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
|
||||
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hf_resolver_with_multiple_repos():
|
||||
hf_resolver = HfHubResolver([LORA_LIB, LORA_REPO])
|
||||
assert hf_resolver is not None
|
||||
|
||||
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
|
||||
assert lora_request is not None
|
||||
assert lora_request.lora_name == LORA_NAME
|
||||
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
|
||||
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_adapter():
|
||||
hf_resolver = HfHubResolver([LORA_LIB])
|
||||
assert hf_resolver is not None
|
||||
|
||||
missing_lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, "foobar")
|
||||
assert missing_lora_request is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonlora_adapter():
|
||||
hf_resolver = HfHubResolver([LORA_LIB])
|
||||
assert hf_resolver is not None
|
||||
|
||||
readme_request = await hf_resolver.resolve_lora(
|
||||
LORA_LIB_MODEL_NAME, NON_LORA_SUBPATH
|
||||
)
|
||||
assert readme_request is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_repo():
|
||||
hf_resolver = HfHubResolver([LORA_LIB])
|
||||
assert hf_resolver is not None
|
||||
|
||||
invalid_repo_req = await hf_resolver.resolve_lora(
|
||||
INVALID_REPO_NAME,
|
||||
f"{INVALID_REPO_NAME}/foo",
|
||||
)
|
||||
assert invalid_repo_req is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trailing_slash():
|
||||
hf_resolver = HfHubResolver([LORA_LIB])
|
||||
assert hf_resolver is not None
|
||||
|
||||
lora_request = await hf_resolver.resolve_lora(
|
||||
LORA_LIB_MODEL_NAME,
|
||||
f"{LORA_NAME}/",
|
||||
)
|
||||
assert lora_request is not None
|
||||
assert lora_request.lora_name == f"{LORA_NAME}/"
|
||||
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
|
||||
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
|
||||
@@ -1,363 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the `vllm.endpoint_plugins` framework (RFC #46565).
|
||||
|
||||
Uses the worked in repo example plugin (`vllm_add_dummy_endpoint_plugin`,
|
||||
installed via `tests/plugins/vllm_add_dummy_endpoint_plugin`) exercising both
|
||||
`EndpointPlugin` hooks against a fake `EngineClient`, unit tests for the
|
||||
`load_endpoint_plugins` gating matrix and an e2e test that drives a real HTTP
|
||||
request through the plugin's route.
|
||||
"""
|
||||
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import APIRouter, FastAPI
|
||||
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
|
||||
|
||||
from vllm.entrypoints.launchers.app import build_app
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.plugins import load_endpoint_plugins
|
||||
from vllm.plugins.endpoint_plugins.interface import (
|
||||
EndpointPlugin,
|
||||
attach_endpoint_plugins,
|
||||
init_endpoint_plugins_state,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class _RaisingEndpointPlugin:
|
||||
"""Factory that raises to exercise the "instantiation fails" path."""
|
||||
|
||||
name = "raising_endpoint_plugin"
|
||||
required_tasks = None
|
||||
|
||||
def __init__(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
|
||||
class _ShadowEndpointPlugin:
|
||||
name = "shadow_endpoint_plugin"
|
||||
required_tasks = None
|
||||
|
||||
def __init__(self, path: str, include_in_schema: bool):
|
||||
self.path = path
|
||||
self.include_in_schema = include_in_schema
|
||||
|
||||
def attach_router(self, app: FastAPI) -> None:
|
||||
assert str(app.url_path_for("show_version")) == "/version"
|
||||
app.openapi()
|
||||
|
||||
@app.get(self.path, status_code=209, include_in_schema=self.include_in_schema)
|
||||
async def plugin_version(plugin_arg: str) -> dict[str, str]:
|
||||
return {"served_by": plugin_arg}
|
||||
|
||||
async def init_state(self, engine_client, state, args) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _RouterEndpointPlugin(DummyAdminEndpointPlugin):
|
||||
def __init__(self, router: APIRouter):
|
||||
self.router = router
|
||||
|
||||
def attach_router(self, app: FastAPI) -> None:
|
||||
app.include_router(self.router)
|
||||
|
||||
|
||||
class _FakeEngineClient:
|
||||
"""Minimal stand in exercising `collective_rpc`. Not a real engine."""
|
||||
|
||||
def __init__(self, rpc_result: Any = None):
|
||||
self.rpc_result = rpc_result
|
||||
self.rpc_calls: list[tuple[str, tuple, dict]] = []
|
||||
|
||||
async def collective_rpc(self, method, timeout=None, args=(), kwargs=None):
|
||||
self.rpc_calls.append((method, args, kwargs or {}))
|
||||
return self.rpc_result
|
||||
|
||||
|
||||
def _build_args() -> Namespace:
|
||||
parser = FlexibleArgumentParser()
|
||||
subparsers = parser.add_subparsers()
|
||||
serve_parser = subparsers.add_parser("serve")
|
||||
make_arg_parser(serve_parser)
|
||||
return serve_parser.parse_args([])
|
||||
|
||||
|
||||
def _fake_loader(factories: dict[str, Any]):
|
||||
def _load_plugins_by_group(group: str) -> dict[str, Any]:
|
||||
assert group == "vllm.endpoint_plugins"
|
||||
return factories
|
||||
|
||||
return _load_plugins_by_group
|
||||
|
||||
|
||||
def test_dummy_plugin_satisfies_protocol():
|
||||
assert isinstance(DummyAdminEndpointPlugin(), EndpointPlugin)
|
||||
|
||||
|
||||
def test_no_plugins_loaded_when_allowlist_unset(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
|
||||
|
||||
assert load_endpoint_plugins(("generate",)) == []
|
||||
|
||||
|
||||
def test_no_plugins_loaded_when_allowlist_is_empty_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""`VLLM_PLUGINS=""` parses to `[""]`, not `None` (see `vllm.envs`), so it
|
||||
must be treated as a (non strict) allowlist matching no plugin name, not
|
||||
as "unset"."""
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "")
|
||||
|
||||
assert load_endpoint_plugins(("generate",)) == []
|
||||
|
||||
|
||||
def test_plugin_loaded_when_allowlisted_and_task_matches(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
|
||||
plugins = load_endpoint_plugins(("generate",))
|
||||
|
||||
assert len(plugins) == 1
|
||||
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
|
||||
|
||||
|
||||
def test_plugin_skipped_when_required_tasks_miss(monkeypatch: pytest.MonkeyPatch):
|
||||
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
|
||||
required_tasks = ("generate",)
|
||||
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_plugins_by_group",
|
||||
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
|
||||
)
|
||||
|
||||
assert load_endpoint_plugins(("embed",)) == []
|
||||
assert len(load_endpoint_plugins(("generate",))) == 1
|
||||
|
||||
|
||||
def test_plugin_loaded_when_required_tasks_is_none(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
|
||||
assert len(load_endpoint_plugins(supported_tasks=None)) == 1
|
||||
|
||||
|
||||
def test_plugin_skipped_when_required_tasks_set_but_supported_tasks_none(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
|
||||
required_tasks = ("generate",)
|
||||
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_plugins_by_group",
|
||||
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
|
||||
)
|
||||
|
||||
assert load_endpoint_plugins(supported_tasks=None) == []
|
||||
|
||||
|
||||
def test_factory_raising_is_logged_and_skipped(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv(
|
||||
"VLLM_PLUGINS", "raising_endpoint_plugin,dummy_admin_endpoint_plugin"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_plugins_by_group",
|
||||
_fake_loader(
|
||||
{
|
||||
"raising_endpoint_plugin": _RaisingEndpointPlugin,
|
||||
"dummy_admin_endpoint_plugin": DummyAdminEndpointPlugin,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
plugins = load_endpoint_plugins(("generate",))
|
||||
|
||||
assert len(plugins) == 1
|
||||
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
|
||||
|
||||
|
||||
def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
|
||||
|
||||
app = FastAPI()
|
||||
attach_endpoint_plugins(app, ("generate",))
|
||||
|
||||
assert app.state.endpoint_plugins == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["/version", "/metrics"])
|
||||
@pytest.mark.parametrize("include_in_schema", [True, False])
|
||||
async def test_endpoint_plugin_override_preserves_hook_and_schema(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, include_in_schema: bool
|
||||
):
|
||||
"""Hooks see core routes; overrides win over both HTTP routes and mounts."""
|
||||
args = _build_args()
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "shadow_endpoint_plugin")
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_plugins_by_group",
|
||||
_fake_loader(
|
||||
{
|
||||
"shadow_endpoint_plugin": lambda: _ShadowEndpointPlugin(
|
||||
path, include_in_schema
|
||||
)
|
||||
}
|
||||
),
|
||||
)
|
||||
app = build_app(args, supported_tasks=())
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get(path, params={"plugin_arg": "plugin"})
|
||||
schema = (await client.get("/openapi.json")).json()
|
||||
|
||||
assert response.status_code == 209
|
||||
assert response.json() == {"served_by": "plugin"}
|
||||
if include_in_schema:
|
||||
operation = schema["paths"][path]["get"]
|
||||
assert "209" in operation["responses"]
|
||||
assert "200" not in operation["responses"]
|
||||
assert operation["parameters"][0]["name"] == "plugin_arg"
|
||||
assert operation["parameters"][0]["required"] is True
|
||||
response_schema = operation["responses"]["209"]["content"]["application/json"][
|
||||
"schema"
|
||||
]
|
||||
assert response_schema["additionalProperties"] == {"type": "string"}
|
||||
else:
|
||||
assert path not in schema["paths"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_plugin_override_preserves_other_methods(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Overriding GET preserves POST's validation and the shared core route."""
|
||||
core = APIRouter()
|
||||
|
||||
@core.api_route("/shared", methods=["GET", "POST"], status_code=201)
|
||||
async def original(value: int) -> dict[str, int]:
|
||||
return {"value": value}
|
||||
|
||||
plugin = APIRouter()
|
||||
|
||||
@plugin.get("/shared", status_code=202)
|
||||
async def replacement():
|
||||
return {"served_by": "plugin"}
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(core)
|
||||
original_route = app.routes[-1]
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_endpoint_plugins",
|
||||
lambda supported_tasks: [_RouterEndpointPlugin(plugin)],
|
||||
)
|
||||
attach_endpoint_plugins(app, ())
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/shared")
|
||||
post_response = await client.post("/shared", params={"value": 7})
|
||||
invalid_response = await client.post("/shared", params={"value": "invalid"})
|
||||
|
||||
assert response.status_code == 202
|
||||
assert response.json() == {"served_by": "plugin"}
|
||||
assert post_response.status_code == 201
|
||||
assert post_response.json() == {"value": 7}
|
||||
assert invalid_response.status_code == 422
|
||||
assert original_route.methods == {"GET", "POST"}
|
||||
assert core.routes[0].methods == {"GET", "POST"}
|
||||
operations = app.openapi()["paths"]["/shared"]
|
||||
assert "202" in operations["get"]["responses"]
|
||||
assert "201" in operations["post"]["responses"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_later_endpoint_plugin_overrides_same_operation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The last registered override wins while unrelated plugin routes survive."""
|
||||
first = APIRouter()
|
||||
|
||||
@first.get("/shared", status_code=201)
|
||||
@first.get("/plugins/first")
|
||||
async def first_handler():
|
||||
return {"served_by": "first"}
|
||||
|
||||
second = APIRouter()
|
||||
|
||||
@second.get("/shared", status_code=202)
|
||||
async def second_handler():
|
||||
return {"served_by": "second"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.plugins.load_endpoint_plugins",
|
||||
lambda supported_tasks: [
|
||||
_RouterEndpointPlugin(first),
|
||||
_RouterEndpointPlugin(second),
|
||||
],
|
||||
)
|
||||
app = FastAPI()
|
||||
attach_endpoint_plugins(app, ())
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/shared")
|
||||
unrelated = await client.get("/plugins/first")
|
||||
unsupported = await client.delete("/shared")
|
||||
|
||||
assert response.status_code == 202
|
||||
assert response.json() == {"served_by": "second"}
|
||||
assert unrelated.status_code == 200
|
||||
assert unrelated.json() == {"served_by": "first"}
|
||||
assert unsupported.status_code == 405
|
||||
responses = app.openapi()["paths"]["/shared"]["get"]["responses"]
|
||||
assert "202" in responses
|
||||
assert "201" not in responses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A caller that builds a bare `State()` without `build_app` must not crash
|
||||
just because `state.endpoint_plugins` was never set."""
|
||||
from starlette.datastructures import State
|
||||
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
|
||||
state = State()
|
||||
await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
|
||||
|
||||
assert not hasattr(state, "dummy_engine_client")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Phase A (attach) + Phase B (init) wired through `build_app` then
|
||||
exercised with a real HTTP request against the worked example plugin."""
|
||||
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
|
||||
|
||||
args = _build_args()
|
||||
app = build_app(args, supported_tasks=())
|
||||
|
||||
assert len(app.state.endpoint_plugins) == 1
|
||||
assert any(
|
||||
getattr(route, "path", None) == "/v1/admin/scheduler_config"
|
||||
for route in app.routes
|
||||
)
|
||||
|
||||
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
|
||||
await init_endpoint_plugins_state(fake_engine_client, app.state, args)
|
||||
|
||||
assert app.state.dummy_engine_client is fake_engine_client
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/v1/admin/scheduler_config")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"scheduler_config": ["cfg-a", "cfg-b"]}
|
||||
assert fake_engine_client.rpc_calls == [("get_scheduler_config", (), {})]
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
|
||||
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
assert chatml_jinja_path.exists()
|
||||
|
||||
|
||||
def run_and_test_dummy_opt_api_server(model, tp=1, extra_args=(), env_dict=None):
|
||||
# the model is registered through the plugin
|
||||
server_args = [
|
||||
"--gpu-memory-utilization",
|
||||
"0.10",
|
||||
"--dtype",
|
||||
"float32",
|
||||
"--chat-template",
|
||||
str(chatml_jinja_path),
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"-tp",
|
||||
f"{tp}",
|
||||
*extra_args,
|
||||
]
|
||||
with RemoteOpenAIServer(model, server_args, env_dict=env_dict) as server:
|
||||
client = server.get_client()
|
||||
completion = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello!"},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
generated_text = completion.choices[0].message.content
|
||||
assert generated_text is not None
|
||||
# make sure only the first token is generated
|
||||
rest = generated_text.replace("<s>", "")
|
||||
assert rest == ""
|
||||
|
||||
|
||||
def test_oot_registration_for_api_server(dummy_opt_path: str):
|
||||
run_and_test_dummy_opt_api_server(dummy_opt_path)
|
||||
@@ -1049,7 +1049,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
|
||||
config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(enable_adaptive_verification=True),
|
||||
lora_config=None,
|
||||
compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
|
||||
)
|
||||
|
||||
@@ -22,7 +22,6 @@ def _create_new_requests_data(prompt_embeds: torch.Tensor | None) -> NewRequestD
|
||||
pooling_params=None,
|
||||
block_ids=([],),
|
||||
num_computed_tokens=0,
|
||||
lora_request=None,
|
||||
prompt_embeds=prompt_embeds,
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ from vllm.distributed.kv_events import (
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
)
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalKwargsItem,
|
||||
@@ -74,7 +73,6 @@ def make_request(
|
||||
mm_hashes: list[str] | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
cache_salt: str | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
session_id: str | None = None,
|
||||
):
|
||||
mm_features = []
|
||||
@@ -98,7 +96,6 @@ def make_request(
|
||||
mm_features=mm_features if mm_features else None,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
lora_request=lora_request,
|
||||
cache_salt=cache_salt,
|
||||
block_hasher=get_request_block_hasher(block_size, hash_fn),
|
||||
session_id=session_id,
|
||||
@@ -2356,64 +2353,6 @@ def test_null_parent_block_hash():
|
||||
assert blocks[num_full_blocks - 1].block_hash is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocks_to_cache", [2, 3, 10])
|
||||
def test_kv_cache_events_with_lora(blocks_to_cache: int):
|
||||
"""Test BlockStored events contain correct lora_id when using LoRA requests."""
|
||||
block_size = 16
|
||||
num_blocks = blocks_to_cache + 1
|
||||
|
||||
# Create KVCacheManager with events enabled
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
enable_kv_cache_events=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
# Test with LoRA request
|
||||
lora_request = LoRARequest(
|
||||
lora_name="test_lora", lora_int_id=42, lora_path="/test/path"
|
||||
)
|
||||
|
||||
num_tokens = block_size * blocks_to_cache
|
||||
req_with_lora = make_request(
|
||||
"lora_req",
|
||||
list(range(num_tokens)),
|
||||
block_size,
|
||||
sha256,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
# Allocate slots and get events
|
||||
_ = manager.allocate_slots(req_with_lora, num_tokens)
|
||||
events = manager.take_events()
|
||||
|
||||
# Verify BlockStored event contains correct lora_id
|
||||
block_stored_event = events[-1]
|
||||
assert isinstance(block_stored_event, BlockStored)
|
||||
assert block_stored_event.lora_id == 42 # Should match lora_request.adapter_id
|
||||
assert len(block_stored_event.block_hashes) == blocks_to_cache
|
||||
assert block_stored_event.block_size == block_size
|
||||
|
||||
# Clean up
|
||||
manager.free(req_with_lora)
|
||||
|
||||
# Test without LoRA request (should have lora_id=None)
|
||||
req_without_lora = make_request(
|
||||
"no_lora_req", list(range(num_tokens)), block_size, sha256
|
||||
)
|
||||
|
||||
_ = manager.allocate_slots(req_without_lora, num_tokens)
|
||||
events = manager.take_events()
|
||||
|
||||
block_stored_event = events[-1]
|
||||
assert isinstance(block_stored_event, BlockStored)
|
||||
assert block_stored_event.lora_id is None # Should be None when no LoRA request
|
||||
assert len(block_stored_event.block_hashes) == blocks_to_cache
|
||||
assert block_stored_event.block_size == block_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_stored_event_group_idx(group_id: int):
|
||||
"""Test BlockStored events emitted by cache_full_blocks carry the correct
|
||||
@@ -2634,9 +2573,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
|
||||
parent_block_hash=None,
|
||||
token_ids=list(range(block_size)),
|
||||
block_size=block_size,
|
||||
lora_id=None,
|
||||
medium=None,
|
||||
lora_name=None,
|
||||
group_idx=1,
|
||||
)
|
||||
manager.block_pool.kv_event_queue.append(event)
|
||||
@@ -2764,8 +2701,6 @@ def test_emit_cached_block_events():
|
||||
assert event.group_idx == kv_cache_group_id
|
||||
assert event.block_size == block_size
|
||||
assert event.medium == MEDIUM_GPU
|
||||
assert event.lora_id is None
|
||||
assert event.lora_name is None
|
||||
assert event.session_id == "agent-session-reuse"
|
||||
|
||||
|
||||
|
||||
@@ -382,9 +382,7 @@ def _make_spec_decode_manager(
|
||||
|
||||
def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
|
||||
manager = _make_spec_decode_manager(monkeypatch)
|
||||
assert [
|
||||
(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[(12, 0)]
|
||||
] == [
|
||||
assert [(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[12]] == [
|
||||
(CUDAGraphMode.FULL, 18),
|
||||
(CUDAGraphMode.PIECEWISE, 16),
|
||||
]
|
||||
@@ -393,7 +391,6 @@ def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
|
||||
num_reqs=4,
|
||||
num_tokens=12,
|
||||
uniform_token_count=_DECODE_QUERY_LEN,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
@@ -409,7 +406,6 @@ def test_uniform_decode_exact_match_is_not_over_padded(monkeypatch):
|
||||
num_reqs=3,
|
||||
num_tokens=9,
|
||||
uniform_token_count=_DECODE_QUERY_LEN,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
@@ -424,7 +420,6 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
|
||||
num_reqs=2,
|
||||
num_tokens=12,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
@@ -443,7 +438,6 @@ def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
|
||||
num_reqs=3,
|
||||
num_tokens=6,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
@@ -479,7 +473,6 @@ def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
|
||||
num_reqs=1,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
)
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE, num_tokens
|
||||
assert desc.uniform_token_count is None, num_tokens
|
||||
@@ -493,7 +486,6 @@ def test_uniform_decode_beyond_capture_ladder_falls_back(monkeypatch):
|
||||
num_reqs=9,
|
||||
num_tokens=27,
|
||||
uniform_token_count=_DECODE_QUERY_LEN,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.NONE
|
||||
@@ -529,13 +521,12 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
||||
|
||||
for desc in full_descs:
|
||||
assert desc.num_reqs is not None
|
||||
assert desc in manager._candidates[(desc.num_tokens, 0)], desc
|
||||
assert desc in manager._candidates[desc.num_tokens], desc
|
||||
assert (
|
||||
manager.dispatch(
|
||||
num_reqs=desc.num_reqs,
|
||||
num_tokens=desc.num_tokens,
|
||||
uniform_token_count=desc.uniform_token_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
== desc
|
||||
), desc
|
||||
@@ -633,7 +624,6 @@ def test_divisor_query_len_dispatch_is_unchanged(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=decode_query_len,
|
||||
num_active_loras=0,
|
||||
)
|
||||
expected = min(s for s in capture_sizes if s >= num_tokens)
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL, num_tokens
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda_alike
|
||||
|
||||
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
|
||||
import vllm.lora.ops.triton_ops.utils as lora_utils
|
||||
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink
|
||||
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def batch_invariant(monkeypatch):
|
||||
monkeypatch.setattr(lora_utils, "is_batch_invariant", True)
|
||||
monkeypatch.setattr(shrink_op, "is_batch_invariant", True)
|
||||
lora_utils.get_lora_op_configs.cache_clear()
|
||||
yield
|
||||
lora_utils.get_lora_op_configs.cache_clear()
|
||||
|
||||
|
||||
def _shrink(inputs, weights, mapping, scaling):
|
||||
num_tokens = inputs.size(0)
|
||||
meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE)
|
||||
meta.prepare_tensors(mapping)
|
||||
out = torch.zeros(
|
||||
(len(weights), num_tokens, weights[0].size(1)),
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
_LORA_A_PTR_DICT.clear()
|
||||
args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False)
|
||||
lora_shrink(inputs, weights, out, *args, scaling)
|
||||
return out
|
||||
|
||||
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,rank,nslices,dtype",
|
||||
[
|
||||
(2560, 8, 3, torch.bfloat16),
|
||||
(9728, 16, 1, torch.float16),
|
||||
(4097, 64, 1, torch.bfloat16), # K tail block
|
||||
(1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty
|
||||
],
|
||||
)
|
||||
def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
|
||||
set_random_seed(0)
|
||||
inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE)
|
||||
weights = [
|
||||
torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE)
|
||||
for _ in range(nslices)
|
||||
]
|
||||
mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE)
|
||||
out = _shrink(inputs, weights, mapping, 0.5).double().cpu()
|
||||
|
||||
ids, x = mapping.cpu(), inputs.double().cpu()
|
||||
ref = torch.zeros_like(out)
|
||||
for s, weight in enumerate(weights):
|
||||
for lora_id in (0, 1):
|
||||
rows = ids == lora_id
|
||||
ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T
|
||||
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("nslices", [1, 3])
|
||||
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
|
||||
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
|
||||
|
||||
class RecordReduce:
|
||||
def __getitem__(self, grid):
|
||||
reduce_grids.append(grid)
|
||||
return reduce_kernel[grid]
|
||||
|
||||
monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce())
|
||||
set_random_seed(0)
|
||||
# K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute.
|
||||
dtype, hidden_size = torch.bfloat16, 2560
|
||||
weights = [
|
||||
torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE)
|
||||
for _ in range(nslices)
|
||||
]
|
||||
target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE)
|
||||
target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
expected = _shrink(target, weights, target_id, 0.7)[:, 0]
|
||||
for num_tokens in (31, 32, 33, 127, 128, 129):
|
||||
inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE)
|
||||
mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE)
|
||||
pos = num_tokens // 2
|
||||
inputs[pos], mapping[pos] = target[0], 0
|
||||
out = _shrink(inputs, weights, mapping, 0.7)
|
||||
assert torch.equal(out[:, pos], expected), num_tokens
|
||||
assert len(reduce_grids) == 7
|
||||
@@ -96,7 +96,6 @@ def _attach_prometheus_logger(
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
vllm_config.ec_transfer_config = None
|
||||
vllm_config.lora_config = None
|
||||
llm.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
engine_idxs=engine_indexes,
|
||||
@@ -128,7 +127,6 @@ def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest:
|
||||
sampling_params=SamplingParams(n=n),
|
||||
pooling_params=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -47,8 +46,6 @@ def _run_dummy_batch(
|
||||
speculator=speculator,
|
||||
kv_connector=SimpleNamespace(set_disabled=lambda disabled: None),
|
||||
is_first_pp_rank=True,
|
||||
lora_config=None,
|
||||
maybe_dummy_run_with_lora=lambda *args, **kwargs: nullcontext(),
|
||||
execute_model=execute_model,
|
||||
)
|
||||
|
||||
|
||||
@@ -137,7 +137,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
# With DSD enabled, MRv2 should have captured a FULL candidate for
|
||||
@@ -146,7 +145,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
assert desc.uniform_token_count == max_query_len
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs == num_reqs
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
|
||||
@@ -216,14 +214,12 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
|
||||
num_reqs=2,
|
||||
num_tokens=3,
|
||||
uniform_token_count=None,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_tokens == 3
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
@@ -276,7 +272,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
@@ -290,7 +285,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
assert desc.cg_mode == CUDAGraphMode.FULL
|
||||
assert desc.uniform_token_count == decode_query_len
|
||||
@@ -344,14 +338,12 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
@@ -409,7 +401,6 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
uniform_token_count=uniform_tok_count,
|
||||
num_active_loras=0,
|
||||
)
|
||||
|
||||
if max_query_len in scheduled_query_lens:
|
||||
@@ -425,4 +416,3 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
assert desc.uniform_token_count is None
|
||||
assert desc.num_tokens == num_tokens
|
||||
assert desc.num_reqs is None
|
||||
assert desc.num_active_loras == 0
|
||||
|
||||
@@ -35,7 +35,6 @@ def mock_model_runner_with_req_states():
|
||||
runner.encoder_cache = None
|
||||
runner.model_state = Mock()
|
||||
runner.block_tables = Mock()
|
||||
runner.lora_state = Mock()
|
||||
runner.pp_handler = None
|
||||
runner.sampler = None
|
||||
runner.prompt_logprobs_worker = None
|
||||
@@ -91,7 +90,6 @@ def test_e2e_streaming_request_update_basic_flow(
|
||||
pooling_params=None,
|
||||
block_ids=([0],),
|
||||
num_computed_tokens=3,
|
||||
lora_request=None,
|
||||
)
|
||||
runner.add_requests(_make_scheduler_output([initial_req_data]))
|
||||
assert req_id in req_states.req_id_to_index
|
||||
@@ -109,7 +107,6 @@ def test_e2e_streaming_request_update_basic_flow(
|
||||
pooling_params=None,
|
||||
block_ids=([0, 1],),
|
||||
num_computed_tokens=4, # 3 original prompt + 1 intermediate output
|
||||
lora_request=None,
|
||||
)
|
||||
runner.add_requests(_make_scheduler_output([updated_req_data]))
|
||||
|
||||
@@ -163,7 +160,6 @@ def test_e2e_streaming_with_multimodal_features(
|
||||
pooling_params=None,
|
||||
block_ids=([0],),
|
||||
num_computed_tokens=14,
|
||||
lora_request=None,
|
||||
)
|
||||
runner.add_requests(_make_scheduler_output([initial_req_data]))
|
||||
assert req_id in req_states.req_id_to_index
|
||||
@@ -185,7 +181,6 @@ def test_e2e_streaming_with_multimodal_features(
|
||||
pooling_params=None,
|
||||
block_ids=([0, 1],),
|
||||
num_computed_tokens=14,
|
||||
lora_request=None,
|
||||
)
|
||||
runner.add_requests(_make_scheduler_output([updated_req_data]))
|
||||
|
||||
|
||||
@@ -270,8 +270,6 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
|
||||
needs_capture=lambda: captured,
|
||||
capture=lambda *args, **kwargs: None,
|
||||
)
|
||||
runner.lora_config = None
|
||||
runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext()
|
||||
runner.speculator = None
|
||||
runner.adaptive_verification = None
|
||||
runner.model = None
|
||||
|
||||
@@ -363,8 +363,6 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch):
|
||||
runner.kv_caches = []
|
||||
runner.attn_groups = []
|
||||
runner.kv_cache_config = SimpleNamespace()
|
||||
runner.lora_config = None
|
||||
runner.maybe_remove_all_loras = lambda _: None
|
||||
allocation_bytes = 64 << 20
|
||||
memory: dict[str, int] = {}
|
||||
|
||||
@@ -432,8 +430,6 @@ def test_teardown_profiling_state_clears_mamba_align_metadata(monkeypatch):
|
||||
runner.attn_groups = []
|
||||
runner.kv_cache_config = SimpleNamespace()
|
||||
runner.cudagraph_manager = object()
|
||||
runner.lora_config = None
|
||||
runner.maybe_remove_all_loras = lambda _: None
|
||||
|
||||
monkeypatch.setattr(cgu.torch.accelerator, "synchronize", lambda: None)
|
||||
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
|
||||
|
||||
@@ -81,7 +81,6 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
load_config=runner.load_config,
|
||||
model_config=runner.model_config,
|
||||
)
|
||||
runner.lora_config = None
|
||||
runner.use_aux_hidden_state_outputs = False
|
||||
runner.speculative_config = None
|
||||
runner.speculator = None
|
||||
|
||||
@@ -8,12 +8,9 @@ session is active. These tests verify that delegation and the session guard.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.lora.layers import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader.reload.inplace import get_parameter_for_reload
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
@@ -57,13 +54,13 @@ class _RecordingEngine:
|
||||
class _RecordingModelRunner:
|
||||
def __init__(self) -> None:
|
||||
self.seen_config: VllmConfig | None = None
|
||||
self.reset_lora_calls = 0
|
||||
self.refresh_draft_weights_calls = 0
|
||||
|
||||
def reload_weights(self) -> None:
|
||||
self.seen_config = get_current_vllm_config()
|
||||
|
||||
def reset_lora_state(self) -> None:
|
||||
self.reset_lora_calls += 1
|
||||
def refresh_draft_weights(self) -> None:
|
||||
self.refresh_draft_weights_calls += 1
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
return None
|
||||
@@ -75,6 +72,7 @@ def _make_worker(engine: _RecordingEngine | None) -> Worker:
|
||||
worker.weight_transfer_engine = engine
|
||||
worker._weight_update_active = False
|
||||
worker._weight_update_is_draft = False
|
||||
worker.use_v2_model_runner = False
|
||||
worker.model_runner = _RecordingModelRunner()
|
||||
return worker
|
||||
|
||||
@@ -108,22 +106,6 @@ def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
|
||||
assert worker.model_runner is model_runner
|
||||
|
||||
|
||||
def test_reload_parameter_lookup_preserves_lora_module_names():
|
||||
base_layer = nn.Module()
|
||||
qweight = nn.Parameter(torch.ones(1))
|
||||
base_layer.register_parameter("qweight", qweight)
|
||||
wrapper = BaseLayerWithLoRA()
|
||||
wrapper.base_layer = base_layer
|
||||
model = nn.Module()
|
||||
model.proj = wrapper
|
||||
|
||||
named_parameters = dict(model.named_parameters())
|
||||
assert set(named_parameters) == {"proj.base_layer.qweight"}
|
||||
assert named_parameters["proj.base_layer.qweight"] is qweight
|
||||
assert model.get_parameter("proj.base_layer.qweight") is qweight
|
||||
assert get_parameter_for_reload(model, "proj.qweight") is qweight
|
||||
|
||||
|
||||
def test_start_update_finish_delegates_to_engine():
|
||||
engine = _RecordingEngine()
|
||||
worker = _make_worker(engine)
|
||||
@@ -141,7 +123,7 @@ def test_start_update_finish_delegates_to_engine():
|
||||
assert engine.reset_count == 1
|
||||
assert worker._weight_update_active is False
|
||||
assert engine.seen_configs == [worker.vllm_config] * 3
|
||||
assert worker.model_runner.reset_lora_calls == 1
|
||||
assert worker.model_runner.refresh_draft_weights_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -194,7 +176,7 @@ def test_rank_local_update_uses_data_parallel_index_after_reconfigure():
|
||||
assert worker._weight_update_active is True
|
||||
|
||||
|
||||
def test_finish_draft_session_keeps_lora_state():
|
||||
def test_finish_draft_session_does_not_refresh_draft_weights():
|
||||
engine = _RecordingEngine()
|
||||
engine.supports_draft_weight_update = True
|
||||
worker = _make_worker(engine)
|
||||
@@ -203,7 +185,7 @@ def test_finish_draft_session_keeps_lora_state():
|
||||
Worker.start_draft_weight_update(worker)
|
||||
Worker.finish_weight_update(worker)
|
||||
|
||||
assert worker.model_runner.reset_lora_calls == 0
|
||||
assert worker.model_runner.refresh_draft_weights_calls == 0
|
||||
|
||||
|
||||
def test_double_start_raises():
|
||||
|
||||
Reference in New Issue
Block a user